2024-11-26 09:55:57,470 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-26 09:55:57,491 main DEBUG Took 0.018826 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-26 09:55:57,491 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-26 09:55:57,491 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-26 09:55:57,492 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-26 09:55:57,494 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:55:57,503 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-26 09:55:57,521 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:55:57,523 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:55:57,524 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:55:57,525 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:55:57,526 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:55:57,526 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:55:57,527 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:55:57,528 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:55:57,528 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:55:57,529 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:55:57,530 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:55:57,530 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:55:57,531 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:55:57,532 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:55:57,532 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:55:57,533 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:55:57,533 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:55:57,534 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:55:57,535 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:55:57,535 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:55:57,536 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:55:57,536 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:55:57,537 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:55:57,537 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:55:57,538 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:55:57,538 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-26 09:55:57,541 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:55:57,542 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-26 09:55:57,545 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-26 09:55:57,546 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-26 09:55:57,548 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-26 09:55:57,548 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-26 09:55:57,561 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-26 09:55:57,565 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-26 09:55:57,568 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-26 09:55:57,568 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-26 09:55:57,569 main DEBUG createAppenders(={Console}) 2024-11-26 09:55:57,570 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b initialized 2024-11-26 09:55:57,570 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-26 09:55:57,571 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b OK. 2024-11-26 09:55:57,571 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-26 09:55:57,572 main DEBUG OutputStream closed 2024-11-26 09:55:57,572 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-26 09:55:57,573 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-26 09:55:57,573 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@37d4349f OK 2024-11-26 09:55:57,658 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-26 09:55:57,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-26 09:55:57,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-26 09:55:57,663 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-26 09:55:57,663 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-26 09:55:57,664 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-26 09:55:57,664 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-26 09:55:57,665 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-26 09:55:57,665 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-26 09:55:57,666 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-26 09:55:57,666 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-26 09:55:57,667 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-26 09:55:57,667 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-26 09:55:57,668 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-26 09:55:57,668 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-26 09:55:57,668 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-26 09:55:57,668 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-26 09:55:57,669 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-26 09:55:57,671 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26 09:55:57,672 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4331d187) with optional ClassLoader: null 2024-11-26 09:55:57,672 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-26 09:55:57,673 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4331d187] started OK. 2024-11-26T09:55:57,688 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync timeout: 13 mins 2024-11-26 09:55:57,692 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-26 09:55:57,692 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26T09:55:57,758 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=11, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=162, ProcessCount=11, AvailableMemoryMB=4076 2024-11-26T09:55:58,033 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de 2024-11-26T09:55:58,716 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-26T09:55:58,831 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-26T09:55:58,894 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=wal, suffix=, logDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync, archiveDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/oldWALs, maxLogs=55 2024-11-26T09:55:58,954 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614958942 2024-11-26T09:55:59,029 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614958942 2024-11-26T09:55:59,124 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,131 INFO [Time-limited test {}] regionserver.ChunkCreator(472): data poolSizePercentage is less than 0. So not using pool 2024-11-26T09:55:59,132 INFO [Time-limited test {}] regionserver.ChunkCreator(472): index poolSizePercentage is less than 0. So not using pool 2024-11-26T09:55:59,179 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => de4af5b36d9054a312d130701461afdb, NAME => 'testLockupAroundBadAssignSync,,1732614959147.de4af5b36d9054a312d130701461afdb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testLockupAroundBadAssignSync', {TABLE_ATTRIBUTES => {DURABILITY => 'SYNC_WAL', READONLY => 'false'}}, {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}}, regionDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de 2024-11-26T09:55:59,234 INFO [Time-limited test {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-26T09:55:59,239 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testLockupAroundBadAssignSync,,1732614959147.de4af5b36d9054a312d130701461afdb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T09:55:59,346 INFO [StoreOpener-de4af5b36d9054a312d130701461afdb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family MyCF of region de4af5b36d9054a312d130701461afdb 2024-11-26T09:55:59,386 INFO [StoreOpener-de4af5b36d9054a312d130701461afdb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de4af5b36d9054a312d130701461afdb columnFamilyName MyCF 2024-11-26T09:55:59,394 DEBUG [StoreOpener-de4af5b36d9054a312d130701461afdb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T09:55:59,401 INFO [StoreOpener-de4af5b36d9054a312d130701461afdb-1 {}] regionserver.HStore(327): Store=de4af5b36d9054a312d130701461afdb/MyCF, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T09:55:59,405 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for de4af5b36d9054a312d130701461afdb 2024-11-26T09:55:59,414 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/data/default/testLockupAroundBadAssignSync/de4af5b36d9054a312d130701461afdb 2024-11-26T09:55:59,415 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/data/default/testLockupAroundBadAssignSync/de4af5b36d9054a312d130701461afdb 2024-11-26T09:55:59,422 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for de4af5b36d9054a312d130701461afdb 2024-11-26T09:55:59,423 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for de4af5b36d9054a312d130701461afdb 2024-11-26T09:55:59,432 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for de4af5b36d9054a312d130701461afdb 2024-11-26T09:55:59,445 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/data/default/testLockupAroundBadAssignSync/de4af5b36d9054a312d130701461afdb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T09:55:59,447 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened de4af5b36d9054a312d130701461afdb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67217941, jitterRate=0.0016253739595413208}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T09:55:59,455 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-26T09:55:59,470 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for de4af5b36d9054a312d130701461afdb: Writing region info on filesystem at 1732614959310Initializing all the Stores at 1732614959312 (+2 ms)Instantiating store for column family {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}} at 1732614959313 (+1 ms)Cleaning up temporary data from old regions at 1732614959423 (+110 ms)Region opened successfully at 1732614959467 (+44 ms) 2024-11-26T09:55:59,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614958942) roll requested 2024-11-26T09:55:59,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959527 2024-11-26T09:55:59,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614958942 with entries=1, filesize=249 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959527 2024-11-26T09:55:59,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959527) roll requested 2024-11-26T09:55:59,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959543 2024-11-26T09:55:59,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959543 2024-11-26T09:55:59,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959543) roll requested 2024-11-26T09:55:59,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959558 2024-11-26T09:55:59,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959558 2024-11-26T09:55:59,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959558) roll requested 2024-11-26T09:55:59,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959571 2024-11-26T09:55:59,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959571 2024-11-26T09:55:59,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959571) roll requested 2024-11-26T09:55:59,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959585 2024-11-26T09:55:59,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959585 2024-11-26T09:55:59,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959585) roll requested 2024-11-26T09:55:59,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959600 2024-11-26T09:55:59,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959600 2024-11-26T09:55:59,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959600) roll requested 2024-11-26T09:55:59,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959616 2024-11-26T09:55:59,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959600 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959616 2024-11-26T09:55:59,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959616) roll requested 2024-11-26T09:55:59,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959630 2024-11-26T09:55:59,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959630 2024-11-26T09:55:59,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959630) roll requested 2024-11-26T09:55:59,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959642 2024-11-26T09:55:59,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959642 2024-11-26T09:55:59,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959642) roll requested 2024-11-26T09:55:59,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959656 2024-11-26T09:55:59,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959656 2024-11-26T09:55:59,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959656) roll requested 2024-11-26T09:55:59,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959674 2024-11-26T09:55:59,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959674 2024-11-26T09:55:59,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959674) roll requested 2024-11-26T09:55:59,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959686 2024-11-26T09:55:59,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959686 2024-11-26T09:55:59,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959686) roll requested 2024-11-26T09:55:59,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959697 2024-11-26T09:55:59,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959697 2024-11-26T09:55:59,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959697) roll requested 2024-11-26T09:55:59,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959708 2024-11-26T09:55:59,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959708 2024-11-26T09:55:59,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959708) roll requested 2024-11-26T09:55:59,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959721 2024-11-26T09:55:59,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959721 2024-11-26T09:55:59,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959721) roll requested 2024-11-26T09:55:59,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959733 2024-11-26T09:55:59,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959733 2024-11-26T09:55:59,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959733) roll requested 2024-11-26T09:55:59,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959744 2024-11-26T09:55:59,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959744 2024-11-26T09:55:59,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959744) roll requested 2024-11-26T09:55:59,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959762 2024-11-26T09:55:59,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959762 2024-11-26T09:55:59,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959762) roll requested 2024-11-26T09:55:59,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959773 2024-11-26T09:55:59,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959773 2024-11-26T09:55:59,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959773) roll requested 2024-11-26T09:55:59,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959784 2024-11-26T09:55:59,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959784 2024-11-26T09:55:59,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959784) roll requested 2024-11-26T09:55:59,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959795 2024-11-26T09:55:59,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959795 2024-11-26T09:55:59,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959795) roll requested 2024-11-26T09:55:59,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959813 2024-11-26T09:55:59,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959813 2024-11-26T09:55:59,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959813) roll requested 2024-11-26T09:55:59,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959826 2024-11-26T09:55:59,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959826 2024-11-26T09:55:59,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959826) roll requested 2024-11-26T09:55:59,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959843 2024-11-26T09:55:59,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959843 2024-11-26T09:55:59,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959843) roll requested 2024-11-26T09:55:59,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959860 2024-11-26T09:55:59,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959860 2024-11-26T09:55:59,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959860) roll requested 2024-11-26T09:55:59,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959874 2024-11-26T09:55:59,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959874 2024-11-26T09:55:59,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959874) roll requested 2024-11-26T09:55:59,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959884 2024-11-26T09:55:59,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959884 2024-11-26T09:55:59,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959884) roll requested 2024-11-26T09:55:59,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959894 2024-11-26T09:55:59,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959894 2024-11-26T09:55:59,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959894) roll requested 2024-11-26T09:55:59,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959903 2024-11-26T09:55:59,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959903 2024-11-26T09:55:59,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959903) roll requested 2024-11-26T09:55:59,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959919 2024-11-26T09:55:59,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959919 2024-11-26T09:55:59,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959919) roll requested 2024-11-26T09:55:59,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959928 2024-11-26T09:55:59,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959928 2024-11-26T09:55:59,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959928) roll requested 2024-11-26T09:55:59,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959943 2024-11-26T09:55:59,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959943 2024-11-26T09:55:59,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959943) roll requested 2024-11-26T09:55:59,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959953 2024-11-26T09:55:59,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959953 2024-11-26T09:55:59,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959953) roll requested 2024-11-26T09:55:59,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959965 2024-11-26T09:55:59,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959965 2024-11-26T09:55:59,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959965) roll requested 2024-11-26T09:55:59,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959974 2024-11-26T09:55:59,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959974 2024-11-26T09:55:59,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959974) roll requested 2024-11-26T09:55:59,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959983 2024-11-26T09:55:59,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:55:59,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959983 2024-11-26T09:55:59,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:55:59,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:55:59,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959983) roll requested 2024-11-26T09:55:59,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614959993 2024-11-26T09:56:00,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959993 2024-11-26T09:56:00,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614959993) roll requested 2024-11-26T09:56:00,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960003 2024-11-26T09:56:00,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614959993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960003 2024-11-26T09:56:00,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960003) roll requested 2024-11-26T09:56:00,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960013 2024-11-26T09:56:00,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960013 2024-11-26T09:56:00,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960013) roll requested 2024-11-26T09:56:00,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960022 2024-11-26T09:56:00,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960022 2024-11-26T09:56:00,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960022) roll requested 2024-11-26T09:56:00,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960032 2024-11-26T09:56:00,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960032 2024-11-26T09:56:00,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960032) roll requested 2024-11-26T09:56:00,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960041 2024-11-26T09:56:00,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960041 2024-11-26T09:56:00,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960041) roll requested 2024-11-26T09:56:00,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960052 2024-11-26T09:56:00,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960052 2024-11-26T09:56:00,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960052) roll requested 2024-11-26T09:56:00,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960063 2024-11-26T09:56:00,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960063 2024-11-26T09:56:00,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960063) roll requested 2024-11-26T09:56:00,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960072 2024-11-26T09:56:00,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960072 2024-11-26T09:56:00,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960072) roll requested 2024-11-26T09:56:00,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960081 2024-11-26T09:56:00,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960081 2024-11-26T09:56:00,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960081) roll requested 2024-11-26T09:56:00,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960090 2024-11-26T09:56:00,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960090 2024-11-26T09:56:00,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960090) roll requested 2024-11-26T09:56:00,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960098 2024-11-26T09:56:00,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960098 2024-11-26T09:56:00,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960098) roll requested 2024-11-26T09:56:00,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960108 2024-11-26T09:56:00,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960108 2024-11-26T09:56:00,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960108) roll requested 2024-11-26T09:56:00,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960126 2024-11-26T09:56:00,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960126 2024-11-26T09:56:00,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960126) roll requested 2024-11-26T09:56:00,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960136 2024-11-26T09:56:00,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960136 2024-11-26T09:56:00,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960136) roll requested 2024-11-26T09:56:00,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960151 2024-11-26T09:56:00,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960151 2024-11-26T09:56:00,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960151) roll requested 2024-11-26T09:56:00,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960166 2024-11-26T09:56:00,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960166 2024-11-26T09:56:00,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960166) roll requested 2024-11-26T09:56:00,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960175 2024-11-26T09:56:00,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960175 2024-11-26T09:56:00,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960175) roll requested 2024-11-26T09:56:00,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960187 2024-11-26T09:56:00,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960187 2024-11-26T09:56:00,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960187) roll requested 2024-11-26T09:56:00,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960196 2024-11-26T09:56:00,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960196 2024-11-26T09:56:00,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960196) roll requested 2024-11-26T09:56:00,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960207 2024-11-26T09:56:00,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960207 2024-11-26T09:56:00,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960207) roll requested 2024-11-26T09:56:00,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960216 2024-11-26T09:56:00,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960216 2024-11-26T09:56:00,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960216) roll requested 2024-11-26T09:56:00,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960225 2024-11-26T09:56:00,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960225 2024-11-26T09:56:00,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960225) roll requested 2024-11-26T09:56:00,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960234 2024-11-26T09:56:00,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960234 2024-11-26T09:56:00,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960234) roll requested 2024-11-26T09:56:00,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960248 2024-11-26T09:56:00,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960248 2024-11-26T09:56:00,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960248) roll requested 2024-11-26T09:56:00,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960258 2024-11-26T09:56:00,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960258 2024-11-26T09:56:00,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960258) roll requested 2024-11-26T09:56:00,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960268 2024-11-26T09:56:00,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960268 2024-11-26T09:56:00,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960268) roll requested 2024-11-26T09:56:00,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960278 2024-11-26T09:56:00,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960278 2024-11-26T09:56:00,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960278) roll requested 2024-11-26T09:56:00,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960287 2024-11-26T09:56:00,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960287 2024-11-26T09:56:00,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960287) roll requested 2024-11-26T09:56:00,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960305 2024-11-26T09:56:00,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960305 2024-11-26T09:56:00,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960305) roll requested 2024-11-26T09:56:00,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960314 2024-11-26T09:56:00,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960314 2024-11-26T09:56:00,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960314) roll requested 2024-11-26T09:56:00,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960323 2024-11-26T09:56:00,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960323 2024-11-26T09:56:00,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960323) roll requested 2024-11-26T09:56:00,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960336 2024-11-26T09:56:00,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960336 2024-11-26T09:56:00,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960336) roll requested 2024-11-26T09:56:00,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960345 2024-11-26T09:56:00,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960345 2024-11-26T09:56:00,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960345) roll requested 2024-11-26T09:56:00,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960354 2024-11-26T09:56:00,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960354 2024-11-26T09:56:00,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960354) roll requested 2024-11-26T09:56:00,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960362 2024-11-26T09:56:00,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960362 2024-11-26T09:56:00,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960362) roll requested 2024-11-26T09:56:00,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960370 2024-11-26T09:56:00,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960370 2024-11-26T09:56:00,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960370) roll requested 2024-11-26T09:56:00,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960379 2024-11-26T09:56:00,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960379 2024-11-26T09:56:00,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960379) roll requested 2024-11-26T09:56:00,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960388 2024-11-26T09:56:00,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960388 2024-11-26T09:56:00,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960388) roll requested 2024-11-26T09:56:00,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960396 2024-11-26T09:56:00,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960396 2024-11-26T09:56:00,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960396) roll requested 2024-11-26T09:56:00,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960406 2024-11-26T09:56:00,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960406 2024-11-26T09:56:00,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960406) roll requested 2024-11-26T09:56:00,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960414 2024-11-26T09:56:00,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960414 2024-11-26T09:56:00,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960414) roll requested 2024-11-26T09:56:00,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960422 2024-11-26T09:56:00,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960422 2024-11-26T09:56:00,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960422) roll requested 2024-11-26T09:56:00,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960431 2024-11-26T09:56:00,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960431 2024-11-26T09:56:00,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960431) roll requested 2024-11-26T09:56:00,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960440 2024-11-26T09:56:00,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960440 2024-11-26T09:56:00,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960440) roll requested 2024-11-26T09:56:00,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960449 2024-11-26T09:56:00,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960449 2024-11-26T09:56:00,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960449) roll requested 2024-11-26T09:56:00,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960457 2024-11-26T09:56:00,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960457 2024-11-26T09:56:00,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960457) roll requested 2024-11-26T09:56:00,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960465 2024-11-26T09:56:00,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960465 2024-11-26T09:56:00,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960465) roll requested 2024-11-26T09:56:00,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960474 2024-11-26T09:56:00,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960474 2024-11-26T09:56:00,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960474) roll requested 2024-11-26T09:56:00,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960482 2024-11-26T09:56:00,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960482 2024-11-26T09:56:00,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960482) roll requested 2024-11-26T09:56:00,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960492 2024-11-26T09:56:00,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960492 2024-11-26T09:56:00,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960492) roll requested 2024-11-26T09:56:00,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960503 2024-11-26T09:56:00,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960503 2024-11-26T09:56:00,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960503) roll requested 2024-11-26T09:56:00,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960511 2024-11-26T09:56:00,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960511 2024-11-26T09:56:00,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960511) roll requested 2024-11-26T09:56:00,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960521 2024-11-26T09:56:00,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960521 2024-11-26T09:56:00,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960521) roll requested 2024-11-26T09:56:00,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960530 2024-11-26T09:56:00,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960530 2024-11-26T09:56:00,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960530) roll requested 2024-11-26T09:56:00,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960539 2024-11-26T09:56:00,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960539 2024-11-26T09:56:00,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960539) roll requested 2024-11-26T09:56:00,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960548 2024-11-26T09:56:00,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960548 2024-11-26T09:56:00,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960548) roll requested 2024-11-26T09:56:00,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960557 2024-11-26T09:56:00,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960557 2024-11-26T09:56:00,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960557) roll requested 2024-11-26T09:56:00,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960577 2024-11-26T09:56:00,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960577 2024-11-26T09:56:00,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960577) roll requested 2024-11-26T09:56:00,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960592 2024-11-26T09:56:00,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960592 2024-11-26T09:56:00,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960592) roll requested 2024-11-26T09:56:00,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960604 2024-11-26T09:56:00,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960604 2024-11-26T09:56:00,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960604) roll requested 2024-11-26T09:56:00,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960614 2024-11-26T09:56:00,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960614 2024-11-26T09:56:00,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960614) roll requested 2024-11-26T09:56:00,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960633 2024-11-26T09:56:00,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960633 2024-11-26T09:56:00,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960633) roll requested 2024-11-26T09:56:00,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960643 2024-11-26T09:56:00,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960643 2024-11-26T09:56:00,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960643) roll requested 2024-11-26T09:56:00,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960652 2024-11-26T09:56:00,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960652 2024-11-26T09:56:00,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960652) roll requested 2024-11-26T09:56:00,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960661 2024-11-26T09:56:00,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960661 2024-11-26T09:56:00,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960661) roll requested 2024-11-26T09:56:00,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960669 2024-11-26T09:56:00,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960669 2024-11-26T09:56:00,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960669) roll requested 2024-11-26T09:56:00,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960677 2024-11-26T09:56:00,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960677 2024-11-26T09:56:00,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960677) roll requested 2024-11-26T09:56:00,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960686 2024-11-26T09:56:00,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960686 2024-11-26T09:56:00,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960686) roll requested 2024-11-26T09:56:00,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960694 2024-11-26T09:56:00,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960694 2024-11-26T09:56:00,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960694) roll requested 2024-11-26T09:56:00,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960703 2024-11-26T09:56:00,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960703 2024-11-26T09:56:00,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960703) roll requested 2024-11-26T09:56:00,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960713 2024-11-26T09:56:00,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960713 2024-11-26T09:56:00,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960713) roll requested 2024-11-26T09:56:00,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960721 2024-11-26T09:56:00,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960721 2024-11-26T09:56:00,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960721) roll requested 2024-11-26T09:56:00,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960729 2024-11-26T09:56:00,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960729 2024-11-26T09:56:00,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960729) roll requested 2024-11-26T09:56:00,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960737 2024-11-26T09:56:00,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960737 2024-11-26T09:56:00,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960737) roll requested 2024-11-26T09:56:00,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960746 2024-11-26T09:56:00,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960746 2024-11-26T09:56:00,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960746) roll requested 2024-11-26T09:56:00,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960755 2024-11-26T09:56:00,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960755 2024-11-26T09:56:00,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960755) roll requested 2024-11-26T09:56:00,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960763 2024-11-26T09:56:00,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960763 2024-11-26T09:56:00,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960763) roll requested 2024-11-26T09:56:00,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960772 2024-11-26T09:56:00,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960772 2024-11-26T09:56:00,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960772) roll requested 2024-11-26T09:56:00,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960780 2024-11-26T09:56:00,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960780 2024-11-26T09:56:00,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960780) roll requested 2024-11-26T09:56:00,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960788 2024-11-26T09:56:00,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960788 2024-11-26T09:56:00,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960788) roll requested 2024-11-26T09:56:00,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960796 2024-11-26T09:56:00,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960796 2024-11-26T09:56:00,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960796) roll requested 2024-11-26T09:56:00,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960805 2024-11-26T09:56:00,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960805 2024-11-26T09:56:00,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960805) roll requested 2024-11-26T09:56:00,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960814 2024-11-26T09:56:00,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960814 2024-11-26T09:56:00,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960814) roll requested 2024-11-26T09:56:00,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960823 2024-11-26T09:56:00,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960823 2024-11-26T09:56:00,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960823) roll requested 2024-11-26T09:56:00,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960832 2024-11-26T09:56:00,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960832 2024-11-26T09:56:00,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960832) roll requested 2024-11-26T09:56:00,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960840 2024-11-26T09:56:00,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960840 2024-11-26T09:56:00,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960840) roll requested 2024-11-26T09:56:00,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960849 2024-11-26T09:56:00,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960849 2024-11-26T09:56:00,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960849) roll requested 2024-11-26T09:56:00,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960858 2024-11-26T09:56:00,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960858 2024-11-26T09:56:00,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960858) roll requested 2024-11-26T09:56:00,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960867 2024-11-26T09:56:00,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960867 2024-11-26T09:56:00,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960867) roll requested 2024-11-26T09:56:00,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960875 2024-11-26T09:56:00,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960875 2024-11-26T09:56:00,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960875) roll requested 2024-11-26T09:56:00,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960885 2024-11-26T09:56:00,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960885 2024-11-26T09:56:00,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960885) roll requested 2024-11-26T09:56:00,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960894 2024-11-26T09:56:00,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960894 2024-11-26T09:56:00,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960894) roll requested 2024-11-26T09:56:00,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960902 2024-11-26T09:56:00,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960902 2024-11-26T09:56:00,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960902) roll requested 2024-11-26T09:56:00,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960913 2024-11-26T09:56:00,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960913 2024-11-26T09:56:00,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960913) roll requested 2024-11-26T09:56:00,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960923 2024-11-26T09:56:00,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960923 2024-11-26T09:56:00,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960923) roll requested 2024-11-26T09:56:00,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960932 2024-11-26T09:56:00,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960932 2024-11-26T09:56:00,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960932) roll requested 2024-11-26T09:56:00,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960940 2024-11-26T09:56:00,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960940 2024-11-26T09:56:00,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960940) roll requested 2024-11-26T09:56:00,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960949 2024-11-26T09:56:00,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960949 2024-11-26T09:56:00,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960949) roll requested 2024-11-26T09:56:00,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960958 2024-11-26T09:56:00,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960958 2024-11-26T09:56:00,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960958) roll requested 2024-11-26T09:56:00,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960966 2024-11-26T09:56:00,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960966 2024-11-26T09:56:00,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960966) roll requested 2024-11-26T09:56:00,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960975 2024-11-26T09:56:00,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960975 2024-11-26T09:56:00,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960975) roll requested 2024-11-26T09:56:00,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960983 2024-11-26T09:56:00,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960983 2024-11-26T09:56:00,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:00,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:00,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960983) roll requested 2024-11-26T09:56:00,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614960991 2024-11-26T09:56:00,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:00,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960991 2024-11-26T09:56:01,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614960991) roll requested 2024-11-26T09:56:01,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961003 2024-11-26T09:56:01,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614960991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961003 2024-11-26T09:56:01,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961003) roll requested 2024-11-26T09:56:01,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961018 2024-11-26T09:56:01,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961018 2024-11-26T09:56:01,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961018) roll requested 2024-11-26T09:56:01,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961042 2024-11-26T09:56:01,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961042 2024-11-26T09:56:01,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961042) roll requested 2024-11-26T09:56:01,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961054 2024-11-26T09:56:01,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961054 2024-11-26T09:56:01,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961054) roll requested 2024-11-26T09:56:01,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961064 2024-11-26T09:56:01,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961064 2024-11-26T09:56:01,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961064) roll requested 2024-11-26T09:56:01,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961086 2024-11-26T09:56:01,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961086 2024-11-26T09:56:01,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961086) roll requested 2024-11-26T09:56:01,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961096 2024-11-26T09:56:01,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961096 2024-11-26T09:56:01,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961096) roll requested 2024-11-26T09:56:01,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961105 2024-11-26T09:56:01,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961105 2024-11-26T09:56:01,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961105) roll requested 2024-11-26T09:56:01,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961114 2024-11-26T09:56:01,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961114 2024-11-26T09:56:01,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961114) roll requested 2024-11-26T09:56:01,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961122 2024-11-26T09:56:01,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961122 2024-11-26T09:56:01,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961122) roll requested 2024-11-26T09:56:01,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961133 2024-11-26T09:56:01,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961133 2024-11-26T09:56:01,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961133) roll requested 2024-11-26T09:56:01,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961141 2024-11-26T09:56:01,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961141 2024-11-26T09:56:01,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961141) roll requested 2024-11-26T09:56:01,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961151 2024-11-26T09:56:01,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961151 2024-11-26T09:56:01,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961151) roll requested 2024-11-26T09:56:01,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961160 2024-11-26T09:56:01,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961160 2024-11-26T09:56:01,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961160) roll requested 2024-11-26T09:56:01,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961169 2024-11-26T09:56:01,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961169 2024-11-26T09:56:01,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961169) roll requested 2024-11-26T09:56:01,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961193 2024-11-26T09:56:01,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961193 2024-11-26T09:56:01,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961193) roll requested 2024-11-26T09:56:01,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961202 2024-11-26T09:56:01,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961202 2024-11-26T09:56:01,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961202) roll requested 2024-11-26T09:56:01,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961215 2024-11-26T09:56:01,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961215 2024-11-26T09:56:01,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961215) roll requested 2024-11-26T09:56:01,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961224 2024-11-26T09:56:01,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961224 2024-11-26T09:56:01,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961224) roll requested 2024-11-26T09:56:01,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961238 2024-11-26T09:56:01,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961238 2024-11-26T09:56:01,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961238) roll requested 2024-11-26T09:56:01,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961247 2024-11-26T09:56:01,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961247 2024-11-26T09:56:01,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961247) roll requested 2024-11-26T09:56:01,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961257 2024-11-26T09:56:01,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961257 2024-11-26T09:56:01,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961257) roll requested 2024-11-26T09:56:01,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961265 2024-11-26T09:56:01,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961265 2024-11-26T09:56:01,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961265) roll requested 2024-11-26T09:56:01,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961273 2024-11-26T09:56:01,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961273 2024-11-26T09:56:01,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961273) roll requested 2024-11-26T09:56:01,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961282 2024-11-26T09:56:01,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961282 2024-11-26T09:56:01,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,290 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961282) roll requested 2024-11-26T09:56:01,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961290 2024-11-26T09:56:01,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961290 2024-11-26T09:56:01,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961290) roll requested 2024-11-26T09:56:01,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961298 2024-11-26T09:56:01,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961298 2024-11-26T09:56:01,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961298) roll requested 2024-11-26T09:56:01,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961307 2024-11-26T09:56:01,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961307 2024-11-26T09:56:01,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961307) roll requested 2024-11-26T09:56:01,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961316 2024-11-26T09:56:01,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961316 2024-11-26T09:56:01,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961316) roll requested 2024-11-26T09:56:01,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961324 2024-11-26T09:56:01,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961324 2024-11-26T09:56:01,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961324) roll requested 2024-11-26T09:56:01,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961333 2024-11-26T09:56:01,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961333 2024-11-26T09:56:01,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961333) roll requested 2024-11-26T09:56:01,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961343 2024-11-26T09:56:01,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961343 2024-11-26T09:56:01,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961343) roll requested 2024-11-26T09:56:01,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961352 2024-11-26T09:56:01,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961352 2024-11-26T09:56:01,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961352) roll requested 2024-11-26T09:56:01,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961360 2024-11-26T09:56:01,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961360 2024-11-26T09:56:01,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961360) roll requested 2024-11-26T09:56:01,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961368 2024-11-26T09:56:01,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961368 2024-11-26T09:56:01,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961368) roll requested 2024-11-26T09:56:01,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961376 2024-11-26T09:56:01,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961376 2024-11-26T09:56:01,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961376) roll requested 2024-11-26T09:56:01,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961386 2024-11-26T09:56:01,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961386 2024-11-26T09:56:01,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961386) roll requested 2024-11-26T09:56:01,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961394 2024-11-26T09:56:01,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961394 2024-11-26T09:56:01,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961394) roll requested 2024-11-26T09:56:01,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961402 2024-11-26T09:56:01,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961402 2024-11-26T09:56:01,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961402) roll requested 2024-11-26T09:56:01,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961411 2024-11-26T09:56:01,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961411 2024-11-26T09:56:01,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961411) roll requested 2024-11-26T09:56:01,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961420 2024-11-26T09:56:01,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961420 2024-11-26T09:56:01,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961420) roll requested 2024-11-26T09:56:01,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961429 2024-11-26T09:56:01,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961429 2024-11-26T09:56:01,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961429) roll requested 2024-11-26T09:56:01,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961437 2024-11-26T09:56:01,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961437 2024-11-26T09:56:01,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961437) roll requested 2024-11-26T09:56:01,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961447 2024-11-26T09:56:01,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961447 2024-11-26T09:56:01,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961447) roll requested 2024-11-26T09:56:01,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961456 2024-11-26T09:56:01,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961456 2024-11-26T09:56:01,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961456) roll requested 2024-11-26T09:56:01,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961465 2024-11-26T09:56:01,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961465 2024-11-26T09:56:01,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961465) roll requested 2024-11-26T09:56:01,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961473 2024-11-26T09:56:01,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961473 2024-11-26T09:56:01,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961473) roll requested 2024-11-26T09:56:01,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961482 2024-11-26T09:56:01,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961482 2024-11-26T09:56:01,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961482) roll requested 2024-11-26T09:56:01,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961491 2024-11-26T09:56:01,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961491 2024-11-26T09:56:01,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961491) roll requested 2024-11-26T09:56:01,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961501 2024-11-26T09:56:01,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961501 2024-11-26T09:56:01,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961501) roll requested 2024-11-26T09:56:01,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961509 2024-11-26T09:56:01,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961509 2024-11-26T09:56:01,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961509) roll requested 2024-11-26T09:56:01,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961517 2024-11-26T09:56:01,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961517 2024-11-26T09:56:01,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961517) roll requested 2024-11-26T09:56:01,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961526 2024-11-26T09:56:01,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961526 2024-11-26T09:56:01,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961526) roll requested 2024-11-26T09:56:01,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961538 2024-11-26T09:56:01,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961538 2024-11-26T09:56:01,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961538) roll requested 2024-11-26T09:56:01,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961549 2024-11-26T09:56:01,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961549 2024-11-26T09:56:01,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961549) roll requested 2024-11-26T09:56:01,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961557 2024-11-26T09:56:01,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961557 2024-11-26T09:56:01,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961557) roll requested 2024-11-26T09:56:01,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961566 2024-11-26T09:56:01,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961566 2024-11-26T09:56:01,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961566) roll requested 2024-11-26T09:56:01,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961574 2024-11-26T09:56:01,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961574 2024-11-26T09:56:01,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961574) roll requested 2024-11-26T09:56:01,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961585 2024-11-26T09:56:01,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961585 2024-11-26T09:56:01,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961585) roll requested 2024-11-26T09:56:01,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961593 2024-11-26T09:56:01,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961593 2024-11-26T09:56:01,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961593) roll requested 2024-11-26T09:56:01,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961602 2024-11-26T09:56:01,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961602 2024-11-26T09:56:01,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961602) roll requested 2024-11-26T09:56:01,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961610 2024-11-26T09:56:01,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961610 2024-11-26T09:56:01,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961610) roll requested 2024-11-26T09:56:01,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961620 2024-11-26T09:56:01,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961620 2024-11-26T09:56:01,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961620) roll requested 2024-11-26T09:56:01,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961629 2024-11-26T09:56:01,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961629 2024-11-26T09:56:01,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961629) roll requested 2024-11-26T09:56:01,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961638 2024-11-26T09:56:01,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961638 2024-11-26T09:56:01,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961638) roll requested 2024-11-26T09:56:01,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961648 2024-11-26T09:56:01,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961648 2024-11-26T09:56:01,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961648) roll requested 2024-11-26T09:56:01,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961658 2024-11-26T09:56:01,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961658 2024-11-26T09:56:01,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961658) roll requested 2024-11-26T09:56:01,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961666 2024-11-26T09:56:01,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961666 2024-11-26T09:56:01,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961666) roll requested 2024-11-26T09:56:01,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961675 2024-11-26T09:56:01,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961675 2024-11-26T09:56:01,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961675) roll requested 2024-11-26T09:56:01,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961685 2024-11-26T09:56:01,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961685 2024-11-26T09:56:01,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961685) roll requested 2024-11-26T09:56:01,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961693 2024-11-26T09:56:01,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961693 2024-11-26T09:56:01,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961693) roll requested 2024-11-26T09:56:01,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961702 2024-11-26T09:56:01,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961702 2024-11-26T09:56:01,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961702) roll requested 2024-11-26T09:56:01,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961710 2024-11-26T09:56:01,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961710 2024-11-26T09:56:01,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961710) roll requested 2024-11-26T09:56:01,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961719 2024-11-26T09:56:01,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961719 2024-11-26T09:56:01,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961719) roll requested 2024-11-26T09:56:01,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961727 2024-11-26T09:56:01,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961727 2024-11-26T09:56:01,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961727) roll requested 2024-11-26T09:56:01,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961736 2024-11-26T09:56:01,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961736 2024-11-26T09:56:01,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961736) roll requested 2024-11-26T09:56:01,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961745 2024-11-26T09:56:01,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961745 2024-11-26T09:56:01,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961745) roll requested 2024-11-26T09:56:01,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961753 2024-11-26T09:56:01,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961753 2024-11-26T09:56:01,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961753) roll requested 2024-11-26T09:56:01,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961762 2024-11-26T09:56:01,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961762 2024-11-26T09:56:01,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961762) roll requested 2024-11-26T09:56:01,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961771 2024-11-26T09:56:01,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961771 2024-11-26T09:56:01,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961771) roll requested 2024-11-26T09:56:01,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961779 2024-11-26T09:56:01,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961779 2024-11-26T09:56:01,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961779) roll requested 2024-11-26T09:56:01,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961786 2024-11-26T09:56:01,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961786 2024-11-26T09:56:01,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961786) roll requested 2024-11-26T09:56:01,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961794 2024-11-26T09:56:01,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961794 2024-11-26T09:56:01,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961794) roll requested 2024-11-26T09:56:01,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961803 2024-11-26T09:56:01,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961803 2024-11-26T09:56:01,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961803) roll requested 2024-11-26T09:56:01,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961811 2024-11-26T09:56:01,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961811 2024-11-26T09:56:01,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961811) roll requested 2024-11-26T09:56:01,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961820 2024-11-26T09:56:01,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961820 2024-11-26T09:56:01,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961820) roll requested 2024-11-26T09:56:01,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961828 2024-11-26T09:56:01,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961828 2024-11-26T09:56:01,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961828) roll requested 2024-11-26T09:56:01,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961836 2024-11-26T09:56:01,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961836 2024-11-26T09:56:01,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961836) roll requested 2024-11-26T09:56:01,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961845 2024-11-26T09:56:01,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961845 2024-11-26T09:56:01,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961845) roll requested 2024-11-26T09:56:01,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961853 2024-11-26T09:56:01,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961853 2024-11-26T09:56:01,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961853) roll requested 2024-11-26T09:56:01,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961865 2024-11-26T09:56:01,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961865 2024-11-26T09:56:01,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961865) roll requested 2024-11-26T09:56:01,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961874 2024-11-26T09:56:01,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961874 2024-11-26T09:56:01,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961874) roll requested 2024-11-26T09:56:01,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961886 2024-11-26T09:56:01,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961886 2024-11-26T09:56:01,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961886) roll requested 2024-11-26T09:56:01,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961894 2024-11-26T09:56:01,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961894 2024-11-26T09:56:01,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961894) roll requested 2024-11-26T09:56:01,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961901 2024-11-26T09:56:01,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961901 2024-11-26T09:56:01,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961901) roll requested 2024-11-26T09:56:01,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961909 2024-11-26T09:56:01,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961909 2024-11-26T09:56:01,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961909) roll requested 2024-11-26T09:56:01,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961917 2024-11-26T09:56:01,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961917 2024-11-26T09:56:01,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961917) roll requested 2024-11-26T09:56:01,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961925 2024-11-26T09:56:01,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961925 2024-11-26T09:56:01,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961925) roll requested 2024-11-26T09:56:01,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961933 2024-11-26T09:56:01,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961933 2024-11-26T09:56:01,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961933) roll requested 2024-11-26T09:56:01,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961942 2024-11-26T09:56:01,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961942 2024-11-26T09:56:01,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961942) roll requested 2024-11-26T09:56:01,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961955 2024-11-26T09:56:01,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961955 2024-11-26T09:56:01,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961955) roll requested 2024-11-26T09:56:01,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961963 2024-11-26T09:56:01,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961963 2024-11-26T09:56:01,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961963) roll requested 2024-11-26T09:56:01,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961972 2024-11-26T09:56:01,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961972 2024-11-26T09:56:01,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961972) roll requested 2024-11-26T09:56:01,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961980 2024-11-26T09:56:01,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961980 2024-11-26T09:56:01,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961980) roll requested 2024-11-26T09:56:01,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961989 2024-11-26T09:56:01,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:01,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961989 2024-11-26T09:56:01,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:01,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:01,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961989) roll requested 2024-11-26T09:56:01,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614961998 2024-11-26T09:56:02,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961998 2024-11-26T09:56:02,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614961998) roll requested 2024-11-26T09:56:02,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962009 2024-11-26T09:56:02,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614961998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962009 2024-11-26T09:56:02,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962009) roll requested 2024-11-26T09:56:02,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962022 2024-11-26T09:56:02,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962022 2024-11-26T09:56:02,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962022) roll requested 2024-11-26T09:56:02,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962031 2024-11-26T09:56:02,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962031 2024-11-26T09:56:02,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962031) roll requested 2024-11-26T09:56:02,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962041 2024-11-26T09:56:02,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962041 2024-11-26T09:56:02,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962041) roll requested 2024-11-26T09:56:02,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962054 2024-11-26T09:56:02,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962054 2024-11-26T09:56:02,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962054) roll requested 2024-11-26T09:56:02,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962066 2024-11-26T09:56:02,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962066 2024-11-26T09:56:02,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962066) roll requested 2024-11-26T09:56:02,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962078 2024-11-26T09:56:02,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962078 2024-11-26T09:56:02,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962078) roll requested 2024-11-26T09:56:02,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962090 2024-11-26T09:56:02,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962090 2024-11-26T09:56:02,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962090) roll requested 2024-11-26T09:56:02,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962099 2024-11-26T09:56:02,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962099 2024-11-26T09:56:02,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962099) roll requested 2024-11-26T09:56:02,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962109 2024-11-26T09:56:02,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962109 2024-11-26T09:56:02,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962109) roll requested 2024-11-26T09:56:02,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962121 2024-11-26T09:56:02,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962121 2024-11-26T09:56:02,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962121) roll requested 2024-11-26T09:56:02,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962129 2024-11-26T09:56:02,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962129 2024-11-26T09:56:02,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962129) roll requested 2024-11-26T09:56:02,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962137 2024-11-26T09:56:02,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962137 2024-11-26T09:56:02,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962137) roll requested 2024-11-26T09:56:02,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962145 2024-11-26T09:56:02,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962145 2024-11-26T09:56:02,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962145) roll requested 2024-11-26T09:56:02,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962156 2024-11-26T09:56:02,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962156 2024-11-26T09:56:02,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962156) roll requested 2024-11-26T09:56:02,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962165 2024-11-26T09:56:02,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962165 2024-11-26T09:56:02,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962165) roll requested 2024-11-26T09:56:02,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962173 2024-11-26T09:56:02,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962173 2024-11-26T09:56:02,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962173) roll requested 2024-11-26T09:56:02,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962181 2024-11-26T09:56:02,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962181 2024-11-26T09:56:02,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962181) roll requested 2024-11-26T09:56:02,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962188 2024-11-26T09:56:02,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962188 2024-11-26T09:56:02,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962188) roll requested 2024-11-26T09:56:02,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962198 2024-11-26T09:56:02,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962198 2024-11-26T09:56:02,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962198) roll requested 2024-11-26T09:56:02,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962205 2024-11-26T09:56:02,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962205 2024-11-26T09:56:02,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962205) roll requested 2024-11-26T09:56:02,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962213 2024-11-26T09:56:02,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962213 2024-11-26T09:56:02,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962213) roll requested 2024-11-26T09:56:02,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962221 2024-11-26T09:56:02,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962221 2024-11-26T09:56:02,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962221) roll requested 2024-11-26T09:56:02,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962229 2024-11-26T09:56:02,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962229 2024-11-26T09:56:02,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962229) roll requested 2024-11-26T09:56:02,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962239 2024-11-26T09:56:02,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962239 2024-11-26T09:56:02,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962239) roll requested 2024-11-26T09:56:02,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962247 2024-11-26T09:56:02,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962247 2024-11-26T09:56:02,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962247) roll requested 2024-11-26T09:56:02,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962255 2024-11-26T09:56:02,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962255 2024-11-26T09:56:02,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962255) roll requested 2024-11-26T09:56:02,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962263 2024-11-26T09:56:02,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962263 2024-11-26T09:56:02,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962263) roll requested 2024-11-26T09:56:02,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962275 2024-11-26T09:56:02,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962275 2024-11-26T09:56:02,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962275) roll requested 2024-11-26T09:56:02,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962283 2024-11-26T09:56:02,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962283 2024-11-26T09:56:02,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962283) roll requested 2024-11-26T09:56:02,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962293 2024-11-26T09:56:02,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962293 2024-11-26T09:56:02,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962293) roll requested 2024-11-26T09:56:02,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962301 2024-11-26T09:56:02,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962301 2024-11-26T09:56:02,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962301) roll requested 2024-11-26T09:56:02,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962309 2024-11-26T09:56:02,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962309 2024-11-26T09:56:02,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962309) roll requested 2024-11-26T09:56:02,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962317 2024-11-26T09:56:02,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962317 2024-11-26T09:56:02,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962317) roll requested 2024-11-26T09:56:02,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962324 2024-11-26T09:56:02,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962324 2024-11-26T09:56:02,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962324) roll requested 2024-11-26T09:56:02,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962332 2024-11-26T09:56:02,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962332 2024-11-26T09:56:02,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962332) roll requested 2024-11-26T09:56:02,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962340 2024-11-26T09:56:02,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962340 2024-11-26T09:56:02,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962340) roll requested 2024-11-26T09:56:02,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962348 2024-11-26T09:56:02,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962348 2024-11-26T09:56:02,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962348) roll requested 2024-11-26T09:56:02,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962356 2024-11-26T09:56:02,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962356 2024-11-26T09:56:02,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962356) roll requested 2024-11-26T09:56:02,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962378 2024-11-26T09:56:02,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962378 2024-11-26T09:56:02,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962378) roll requested 2024-11-26T09:56:02,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962396 2024-11-26T09:56:02,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962396 2024-11-26T09:56:02,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962396) roll requested 2024-11-26T09:56:02,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962404 2024-11-26T09:56:02,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962404 2024-11-26T09:56:02,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962404) roll requested 2024-11-26T09:56:02,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962413 2024-11-26T09:56:02,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962413 2024-11-26T09:56:02,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962413) roll requested 2024-11-26T09:56:02,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962421 2024-11-26T09:56:02,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962421 2024-11-26T09:56:02,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962421) roll requested 2024-11-26T09:56:02,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962428 2024-11-26T09:56:02,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962428 2024-11-26T09:56:02,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962428) roll requested 2024-11-26T09:56:02,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962436 2024-11-26T09:56:02,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962436 2024-11-26T09:56:02,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962436) roll requested 2024-11-26T09:56:02,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962444 2024-11-26T09:56:02,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962444 2024-11-26T09:56:02,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962444) roll requested 2024-11-26T09:56:02,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962454 2024-11-26T09:56:02,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962454 2024-11-26T09:56:02,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962454) roll requested 2024-11-26T09:56:02,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962464 2024-11-26T09:56:02,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962464 2024-11-26T09:56:02,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962464) roll requested 2024-11-26T09:56:02,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962473 2024-11-26T09:56:02,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962473 2024-11-26T09:56:02,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962473) roll requested 2024-11-26T09:56:02,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962485 2024-11-26T09:56:02,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962485 2024-11-26T09:56:02,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962485) roll requested 2024-11-26T09:56:02,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962502 2024-11-26T09:56:02,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962502 2024-11-26T09:56:02,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962502) roll requested 2024-11-26T09:56:02,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962511 2024-11-26T09:56:02,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962511 2024-11-26T09:56:02,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962511) roll requested 2024-11-26T09:56:02,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962521 2024-11-26T09:56:02,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962521 2024-11-26T09:56:02,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962521) roll requested 2024-11-26T09:56:02,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962529 2024-11-26T09:56:02,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962529 2024-11-26T09:56:02,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962529) roll requested 2024-11-26T09:56:02,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962537 2024-11-26T09:56:02,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962537 2024-11-26T09:56:02,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962537) roll requested 2024-11-26T09:56:02,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962552 2024-11-26T09:56:02,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962552 2024-11-26T09:56:02,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962552) roll requested 2024-11-26T09:56:02,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962559 2024-11-26T09:56:02,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962559 2024-11-26T09:56:02,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962559) roll requested 2024-11-26T09:56:02,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962567 2024-11-26T09:56:02,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962567 2024-11-26T09:56:02,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962567) roll requested 2024-11-26T09:56:02,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962576 2024-11-26T09:56:02,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962576 2024-11-26T09:56:02,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962576) roll requested 2024-11-26T09:56:02,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962583 2024-11-26T09:56:02,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962583 2024-11-26T09:56:02,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962583) roll requested 2024-11-26T09:56:02,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962591 2024-11-26T09:56:02,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962591 2024-11-26T09:56:02,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962591) roll requested 2024-11-26T09:56:02,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962599 2024-11-26T09:56:02,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962599 2024-11-26T09:56:02,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962599) roll requested 2024-11-26T09:56:02,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962606 2024-11-26T09:56:02,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962606 2024-11-26T09:56:02,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962606) roll requested 2024-11-26T09:56:02,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962614 2024-11-26T09:56:02,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962614 2024-11-26T09:56:02,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962614) roll requested 2024-11-26T09:56:02,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962623 2024-11-26T09:56:02,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962623 2024-11-26T09:56:02,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962623) roll requested 2024-11-26T09:56:02,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962631 2024-11-26T09:56:02,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962631 2024-11-26T09:56:02,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962631) roll requested 2024-11-26T09:56:02,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962644 2024-11-26T09:56:02,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962644 2024-11-26T09:56:02,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962644) roll requested 2024-11-26T09:56:02,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962652 2024-11-26T09:56:02,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962652 2024-11-26T09:56:02,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962652) roll requested 2024-11-26T09:56:02,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962660 2024-11-26T09:56:02,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962660 2024-11-26T09:56:02,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962660) roll requested 2024-11-26T09:56:02,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962668 2024-11-26T09:56:02,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962668 2024-11-26T09:56:02,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962668) roll requested 2024-11-26T09:56:02,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962675 2024-11-26T09:56:02,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962675 2024-11-26T09:56:02,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962675) roll requested 2024-11-26T09:56:02,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962683 2024-11-26T09:56:02,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962683 2024-11-26T09:56:02,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962683) roll requested 2024-11-26T09:56:02,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962693 2024-11-26T09:56:02,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962693 2024-11-26T09:56:02,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962693) roll requested 2024-11-26T09:56:02,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962700 2024-11-26T09:56:02,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962700 2024-11-26T09:56:02,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962700) roll requested 2024-11-26T09:56:02,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962711 2024-11-26T09:56:02,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962711 2024-11-26T09:56:02,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962711) roll requested 2024-11-26T09:56:02,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962719 2024-11-26T09:56:02,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962719 2024-11-26T09:56:02,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962719) roll requested 2024-11-26T09:56:02,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962726 2024-11-26T09:56:02,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962726 2024-11-26T09:56:02,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962726) roll requested 2024-11-26T09:56:02,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962733 2024-11-26T09:56:02,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962733 2024-11-26T09:56:02,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962733) roll requested 2024-11-26T09:56:02,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962741 2024-11-26T09:56:02,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962741 2024-11-26T09:56:02,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962741) roll requested 2024-11-26T09:56:02,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962749 2024-11-26T09:56:02,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962749 2024-11-26T09:56:02,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962749) roll requested 2024-11-26T09:56:02,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962756 2024-11-26T09:56:02,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962756 2024-11-26T09:56:02,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962756) roll requested 2024-11-26T09:56:02,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962763 2024-11-26T09:56:02,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962763 2024-11-26T09:56:02,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962763) roll requested 2024-11-26T09:56:02,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962776 2024-11-26T09:56:02,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962776 2024-11-26T09:56:02,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962776) roll requested 2024-11-26T09:56:02,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962784 2024-11-26T09:56:02,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962784 2024-11-26T09:56:02,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962784) roll requested 2024-11-26T09:56:02,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962792 2024-11-26T09:56:02,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962792 2024-11-26T09:56:02,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962792) roll requested 2024-11-26T09:56:02,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962805 2024-11-26T09:56:02,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962805 2024-11-26T09:56:02,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962805) roll requested 2024-11-26T09:56:02,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962813 2024-11-26T09:56:02,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962813 2024-11-26T09:56:02,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962813) roll requested 2024-11-26T09:56:02,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962821 2024-11-26T09:56:02,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962821 2024-11-26T09:56:02,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962821) roll requested 2024-11-26T09:56:02,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962829 2024-11-26T09:56:02,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962829 2024-11-26T09:56:02,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962829) roll requested 2024-11-26T09:56:02,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962837 2024-11-26T09:56:02,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962837 2024-11-26T09:56:02,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962837) roll requested 2024-11-26T09:56:02,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962845 2024-11-26T09:56:02,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962845 2024-11-26T09:56:02,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962845) roll requested 2024-11-26T09:56:02,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962852 2024-11-26T09:56:02,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962852 2024-11-26T09:56:02,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962852) roll requested 2024-11-26T09:56:02,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962860 2024-11-26T09:56:02,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962860 2024-11-26T09:56:02,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962860) roll requested 2024-11-26T09:56:02,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962867 2024-11-26T09:56:02,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962867 2024-11-26T09:56:02,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962867) roll requested 2024-11-26T09:56:02,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962876 2024-11-26T09:56:02,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962876 2024-11-26T09:56:02,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962876) roll requested 2024-11-26T09:56:02,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962883 2024-11-26T09:56:02,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962883 2024-11-26T09:56:02,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962883) roll requested 2024-11-26T09:56:02,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962891 2024-11-26T09:56:02,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962891 2024-11-26T09:56:02,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962891) roll requested 2024-11-26T09:56:02,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962898 2024-11-26T09:56:02,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962898 2024-11-26T09:56:02,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962898) roll requested 2024-11-26T09:56:02,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962906 2024-11-26T09:56:02,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962906 2024-11-26T09:56:02,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962906) roll requested 2024-11-26T09:56:02,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962920 2024-11-26T09:56:02,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962920 2024-11-26T09:56:02,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962920) roll requested 2024-11-26T09:56:02,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962927 2024-11-26T09:56:02,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962927 2024-11-26T09:56:02,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962927) roll requested 2024-11-26T09:56:02,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962936 2024-11-26T09:56:02,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962936 2024-11-26T09:56:02,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962936) roll requested 2024-11-26T09:56:02,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962957 2024-11-26T09:56:02,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962957 2024-11-26T09:56:02,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962957) roll requested 2024-11-26T09:56:02,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962974 2024-11-26T09:56:02,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962974 2024-11-26T09:56:02,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962974) roll requested 2024-11-26T09:56:02,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962982 2024-11-26T09:56:02,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962982 2024-11-26T09:56:02,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962982) roll requested 2024-11-26T09:56:02,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962991 2024-11-26T09:56:02,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:02,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962991 2024-11-26T09:56:02,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:02,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:02,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962991) roll requested 2024-11-26T09:56:02,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614962999 2024-11-26T09:56:03,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962999 2024-11-26T09:56:03,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614962999) roll requested 2024-11-26T09:56:03,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963007 2024-11-26T09:56:03,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614962999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963007 2024-11-26T09:56:03,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963007) roll requested 2024-11-26T09:56:03,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963017 2024-11-26T09:56:03,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963017 2024-11-26T09:56:03,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963017) roll requested 2024-11-26T09:56:03,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963025 2024-11-26T09:56:03,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963025 2024-11-26T09:56:03,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963025) roll requested 2024-11-26T09:56:03,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963033 2024-11-26T09:56:03,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963033 2024-11-26T09:56:03,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963033) roll requested 2024-11-26T09:56:03,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963041 2024-11-26T09:56:03,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963041 2024-11-26T09:56:03,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963041) roll requested 2024-11-26T09:56:03,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963049 2024-11-26T09:56:03,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963049 2024-11-26T09:56:03,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963049) roll requested 2024-11-26T09:56:03,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963057 2024-11-26T09:56:03,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963057 2024-11-26T09:56:03,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963057) roll requested 2024-11-26T09:56:03,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963065 2024-11-26T09:56:03,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963065 2024-11-26T09:56:03,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963065) roll requested 2024-11-26T09:56:03,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963073 2024-11-26T09:56:03,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963073 2024-11-26T09:56:03,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963073) roll requested 2024-11-26T09:56:03,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963081 2024-11-26T09:56:03,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963081 2024-11-26T09:56:03,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963081) roll requested 2024-11-26T09:56:03,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963089 2024-11-26T09:56:03,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963089 2024-11-26T09:56:03,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963089) roll requested 2024-11-26T09:56:03,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963106 2024-11-26T09:56:03,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963106 2024-11-26T09:56:03,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963106) roll requested 2024-11-26T09:56:03,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963118 2024-11-26T09:56:03,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963118 2024-11-26T09:56:03,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963118) roll requested 2024-11-26T09:56:03,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963126 2024-11-26T09:56:03,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963126 2024-11-26T09:56:03,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963126) roll requested 2024-11-26T09:56:03,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963134 2024-11-26T09:56:03,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963134 2024-11-26T09:56:03,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963134) roll requested 2024-11-26T09:56:03,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963142 2024-11-26T09:56:03,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963142 2024-11-26T09:56:03,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963142) roll requested 2024-11-26T09:56:03,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963150 2024-11-26T09:56:03,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963150 2024-11-26T09:56:03,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963150) roll requested 2024-11-26T09:56:03,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963159 2024-11-26T09:56:03,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963159 2024-11-26T09:56:03,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963159) roll requested 2024-11-26T09:56:03,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963166 2024-11-26T09:56:03,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963166 2024-11-26T09:56:03,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963166) roll requested 2024-11-26T09:56:03,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963173 2024-11-26T09:56:03,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963173 2024-11-26T09:56:03,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963173) roll requested 2024-11-26T09:56:03,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963181 2024-11-26T09:56:03,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963181 2024-11-26T09:56:03,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963181) roll requested 2024-11-26T09:56:03,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963189 2024-11-26T09:56:03,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963189 2024-11-26T09:56:03,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963189) roll requested 2024-11-26T09:56:03,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963196 2024-11-26T09:56:03,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963196 2024-11-26T09:56:03,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963196) roll requested 2024-11-26T09:56:03,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963203 2024-11-26T09:56:03,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963203 2024-11-26T09:56:03,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963203) roll requested 2024-11-26T09:56:03,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963212 2024-11-26T09:56:03,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963212 2024-11-26T09:56:03,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963212) roll requested 2024-11-26T09:56:03,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963226 2024-11-26T09:56:03,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963226 2024-11-26T09:56:03,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963226) roll requested 2024-11-26T09:56:03,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963241 2024-11-26T09:56:03,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963241 2024-11-26T09:56:03,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963241) roll requested 2024-11-26T09:56:03,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963259 2024-11-26T09:56:03,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963259 2024-11-26T09:56:03,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963259) roll requested 2024-11-26T09:56:03,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963267 2024-11-26T09:56:03,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963267 2024-11-26T09:56:03,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963267) roll requested 2024-11-26T09:56:03,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963277 2024-11-26T09:56:03,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963277 2024-11-26T09:56:03,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963277) roll requested 2024-11-26T09:56:03,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963287 2024-11-26T09:56:03,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963287 2024-11-26T09:56:03,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963287) roll requested 2024-11-26T09:56:03,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963296 2024-11-26T09:56:03,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963296 2024-11-26T09:56:03,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963296) roll requested 2024-11-26T09:56:03,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963303 2024-11-26T09:56:03,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963303 2024-11-26T09:56:03,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963303) roll requested 2024-11-26T09:56:03,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963311 2024-11-26T09:56:03,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963311 2024-11-26T09:56:03,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963311) roll requested 2024-11-26T09:56:03,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963318 2024-11-26T09:56:03,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963318 2024-11-26T09:56:03,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963318) roll requested 2024-11-26T09:56:03,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963326 2024-11-26T09:56:03,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963326 2024-11-26T09:56:03,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963326) roll requested 2024-11-26T09:56:03,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963334 2024-11-26T09:56:03,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963334 2024-11-26T09:56:03,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963334) roll requested 2024-11-26T09:56:03,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963348 2024-11-26T09:56:03,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963348 2024-11-26T09:56:03,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963348) roll requested 2024-11-26T09:56:03,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963356 2024-11-26T09:56:03,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963356 2024-11-26T09:56:03,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963356) roll requested 2024-11-26T09:56:03,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963363 2024-11-26T09:56:03,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963363 2024-11-26T09:56:03,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963363) roll requested 2024-11-26T09:56:03,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963371 2024-11-26T09:56:03,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963371 2024-11-26T09:56:03,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963371) roll requested 2024-11-26T09:56:03,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963378 2024-11-26T09:56:03,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963378 2024-11-26T09:56:03,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963378) roll requested 2024-11-26T09:56:03,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963385 2024-11-26T09:56:03,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963385 2024-11-26T09:56:03,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963385) roll requested 2024-11-26T09:56:03,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963392 2024-11-26T09:56:03,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963392 2024-11-26T09:56:03,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963392) roll requested 2024-11-26T09:56:03,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963399 2024-11-26T09:56:03,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963399 2024-11-26T09:56:03,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963399) roll requested 2024-11-26T09:56:03,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963407 2024-11-26T09:56:03,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963407 2024-11-26T09:56:03,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963407) roll requested 2024-11-26T09:56:03,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963414 2024-11-26T09:56:03,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963414 2024-11-26T09:56:03,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963414) roll requested 2024-11-26T09:56:03,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963422 2024-11-26T09:56:03,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963422 2024-11-26T09:56:03,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963422) roll requested 2024-11-26T09:56:03,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963429 2024-11-26T09:56:03,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963429 2024-11-26T09:56:03,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963429) roll requested 2024-11-26T09:56:03,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963437 2024-11-26T09:56:03,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963437 2024-11-26T09:56:03,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963437) roll requested 2024-11-26T09:56:03,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963444 2024-11-26T09:56:03,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963444 2024-11-26T09:56:03,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963444) roll requested 2024-11-26T09:56:03,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963451 2024-11-26T09:56:03,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963451 2024-11-26T09:56:03,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963451) roll requested 2024-11-26T09:56:03,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963459 2024-11-26T09:56:03,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963459 2024-11-26T09:56:03,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963459) roll requested 2024-11-26T09:56:03,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963466 2024-11-26T09:56:03,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963466 2024-11-26T09:56:03,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963466) roll requested 2024-11-26T09:56:03,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963474 2024-11-26T09:56:03,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963474 2024-11-26T09:56:03,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963474) roll requested 2024-11-26T09:56:03,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963481 2024-11-26T09:56:03,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963481 2024-11-26T09:56:03,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963481) roll requested 2024-11-26T09:56:03,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963495 2024-11-26T09:56:03,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963495 2024-11-26T09:56:03,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963495) roll requested 2024-11-26T09:56:03,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963503 2024-11-26T09:56:03,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963503 2024-11-26T09:56:03,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963503) roll requested 2024-11-26T09:56:03,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963511 2024-11-26T09:56:03,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963511 2024-11-26T09:56:03,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963511) roll requested 2024-11-26T09:56:03,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963518 2024-11-26T09:56:03,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963518 2024-11-26T09:56:03,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963518) roll requested 2024-11-26T09:56:03,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963525 2024-11-26T09:56:03,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963525 2024-11-26T09:56:03,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963525) roll requested 2024-11-26T09:56:03,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963537 2024-11-26T09:56:03,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963537 2024-11-26T09:56:03,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963537) roll requested 2024-11-26T09:56:03,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963544 2024-11-26T09:56:03,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963544 2024-11-26T09:56:03,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963544) roll requested 2024-11-26T09:56:03,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963552 2024-11-26T09:56:03,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963552 2024-11-26T09:56:03,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963552) roll requested 2024-11-26T09:56:03,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963563 2024-11-26T09:56:03,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963563 2024-11-26T09:56:03,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963563) roll requested 2024-11-26T09:56:03,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963570 2024-11-26T09:56:03,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963570 2024-11-26T09:56:03,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963570) roll requested 2024-11-26T09:56:03,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963580 2024-11-26T09:56:03,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963580 2024-11-26T09:56:03,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963580) roll requested 2024-11-26T09:56:03,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963587 2024-11-26T09:56:03,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963587 2024-11-26T09:56:03,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963587) roll requested 2024-11-26T09:56:03,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963594 2024-11-26T09:56:03,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963594 2024-11-26T09:56:03,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963594) roll requested 2024-11-26T09:56:03,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963601 2024-11-26T09:56:03,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963601 2024-11-26T09:56:03,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963601) roll requested 2024-11-26T09:56:03,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963608 2024-11-26T09:56:03,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963608 2024-11-26T09:56:03,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963608) roll requested 2024-11-26T09:56:03,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963616 2024-11-26T09:56:03,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963616 2024-11-26T09:56:03,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963616) roll requested 2024-11-26T09:56:03,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963624 2024-11-26T09:56:03,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963624 2024-11-26T09:56:03,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963624) roll requested 2024-11-26T09:56:03,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963631 2024-11-26T09:56:03,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963631 2024-11-26T09:56:03,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963631) roll requested 2024-11-26T09:56:03,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963641 2024-11-26T09:56:03,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963641 2024-11-26T09:56:03,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963641) roll requested 2024-11-26T09:56:03,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963648 2024-11-26T09:56:03,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963648 2024-11-26T09:56:03,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963648) roll requested 2024-11-26T09:56:03,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963656 2024-11-26T09:56:03,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963656 2024-11-26T09:56:03,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963656) roll requested 2024-11-26T09:56:03,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963664 2024-11-26T09:56:03,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963664 2024-11-26T09:56:03,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963664) roll requested 2024-11-26T09:56:03,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963672 2024-11-26T09:56:03,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963672 2024-11-26T09:56:03,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963672) roll requested 2024-11-26T09:56:03,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963679 2024-11-26T09:56:03,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963679 2024-11-26T09:56:03,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963679) roll requested 2024-11-26T09:56:03,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963687 2024-11-26T09:56:03,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963687 2024-11-26T09:56:03,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963687) roll requested 2024-11-26T09:56:03,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963694 2024-11-26T09:56:03,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963694 2024-11-26T09:56:03,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963694) roll requested 2024-11-26T09:56:03,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963702 2024-11-26T09:56:03,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963702 2024-11-26T09:56:03,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963702) roll requested 2024-11-26T09:56:03,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963709 2024-11-26T09:56:03,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963709 2024-11-26T09:56:03,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963709) roll requested 2024-11-26T09:56:03,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963717 2024-11-26T09:56:03,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963717 2024-11-26T09:56:03,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963717) roll requested 2024-11-26T09:56:03,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963724 2024-11-26T09:56:03,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963724 2024-11-26T09:56:03,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963724) roll requested 2024-11-26T09:56:03,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963731 2024-11-26T09:56:03,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963731 2024-11-26T09:56:03,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,738 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963731) roll requested 2024-11-26T09:56:03,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963738 2024-11-26T09:56:03,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963731 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963738 2024-11-26T09:56:03,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963738) roll requested 2024-11-26T09:56:03,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963746 2024-11-26T09:56:03,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963746 2024-11-26T09:56:03,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963746) roll requested 2024-11-26T09:56:03,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963753 2024-11-26T09:56:03,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963753 2024-11-26T09:56:03,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963753) roll requested 2024-11-26T09:56:03,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963760 2024-11-26T09:56:03,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963760 2024-11-26T09:56:03,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963760) roll requested 2024-11-26T09:56:03,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963767 2024-11-26T09:56:03,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963767 2024-11-26T09:56:03,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963767) roll requested 2024-11-26T09:56:03,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963775 2024-11-26T09:56:03,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963775 2024-11-26T09:56:03,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963775) roll requested 2024-11-26T09:56:03,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963783 2024-11-26T09:56:03,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963783 2024-11-26T09:56:03,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963783) roll requested 2024-11-26T09:56:03,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963791 2024-11-26T09:56:03,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963791 2024-11-26T09:56:03,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963791) roll requested 2024-11-26T09:56:03,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963799 2024-11-26T09:56:03,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963799 2024-11-26T09:56:03,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963799) roll requested 2024-11-26T09:56:03,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963806 2024-11-26T09:56:03,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963806 2024-11-26T09:56:03,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963806) roll requested 2024-11-26T09:56:03,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963821 2024-11-26T09:56:03,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963821 2024-11-26T09:56:03,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963821) roll requested 2024-11-26T09:56:03,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963829 2024-11-26T09:56:03,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963829 2024-11-26T09:56:03,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963829) roll requested 2024-11-26T09:56:03,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963837 2024-11-26T09:56:03,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963837 2024-11-26T09:56:03,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963837) roll requested 2024-11-26T09:56:03,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963844 2024-11-26T09:56:03,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963844 2024-11-26T09:56:03,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963844) roll requested 2024-11-26T09:56:03,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963852 2024-11-26T09:56:03,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963852 2024-11-26T09:56:03,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963852) roll requested 2024-11-26T09:56:03,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963860 2024-11-26T09:56:03,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963860 2024-11-26T09:56:03,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963860) roll requested 2024-11-26T09:56:03,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963868 2024-11-26T09:56:03,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963868 2024-11-26T09:56:03,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963868) roll requested 2024-11-26T09:56:03,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963875 2024-11-26T09:56:03,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963875 2024-11-26T09:56:03,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963875) roll requested 2024-11-26T09:56:03,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963894 2024-11-26T09:56:03,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963894 2024-11-26T09:56:03,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963894) roll requested 2024-11-26T09:56:03,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963903 2024-11-26T09:56:03,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963903 2024-11-26T09:56:03,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963903) roll requested 2024-11-26T09:56:03,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963912 2024-11-26T09:56:03,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963912 2024-11-26T09:56:03,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963912) roll requested 2024-11-26T09:56:03,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963920 2024-11-26T09:56:03,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963920 2024-11-26T09:56:03,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963920) roll requested 2024-11-26T09:56:03,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963927 2024-11-26T09:56:03,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963927 2024-11-26T09:56:03,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963927) roll requested 2024-11-26T09:56:03,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963935 2024-11-26T09:56:03,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963935 2024-11-26T09:56:03,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963935) roll requested 2024-11-26T09:56:03,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963942 2024-11-26T09:56:03,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963942 2024-11-26T09:56:03,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963942) roll requested 2024-11-26T09:56:03,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963952 2024-11-26T09:56:03,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963952 2024-11-26T09:56:03,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963952) roll requested 2024-11-26T09:56:03,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963960 2024-11-26T09:56:03,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963960 2024-11-26T09:56:03,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963960) roll requested 2024-11-26T09:56:03,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963967 2024-11-26T09:56:03,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963967 2024-11-26T09:56:03,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963967) roll requested 2024-11-26T09:56:03,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963975 2024-11-26T09:56:03,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963975 2024-11-26T09:56:03,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963975) roll requested 2024-11-26T09:56:03,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963982 2024-11-26T09:56:03,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963982 2024-11-26T09:56:03,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:03,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:03,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963982) roll requested 2024-11-26T09:56:03,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614963990 2024-11-26T09:56:03,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:03,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963990 2024-11-26T09:56:04,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614963990) roll requested 2024-11-26T09:56:04,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964000 2024-11-26T09:56:04,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614963990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964000 2024-11-26T09:56:04,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964000) roll requested 2024-11-26T09:56:04,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964007 2024-11-26T09:56:04,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964007 2024-11-26T09:56:04,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964007) roll requested 2024-11-26T09:56:04,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964015 2024-11-26T09:56:04,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964015 2024-11-26T09:56:04,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964015) roll requested 2024-11-26T09:56:04,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964024 2024-11-26T09:56:04,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964024 2024-11-26T09:56:04,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964024) roll requested 2024-11-26T09:56:04,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964032 2024-11-26T09:56:04,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964032 2024-11-26T09:56:04,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964032) roll requested 2024-11-26T09:56:04,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964040 2024-11-26T09:56:04,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964040 2024-11-26T09:56:04,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964040) roll requested 2024-11-26T09:56:04,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964048 2024-11-26T09:56:04,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964048 2024-11-26T09:56:04,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964048) roll requested 2024-11-26T09:56:04,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964055 2024-11-26T09:56:04,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964055 2024-11-26T09:56:04,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964055) roll requested 2024-11-26T09:56:04,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964062 2024-11-26T09:56:04,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964062 2024-11-26T09:56:04,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964062) roll requested 2024-11-26T09:56:04,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964070 2024-11-26T09:56:04,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964070 2024-11-26T09:56:04,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964070) roll requested 2024-11-26T09:56:04,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964077 2024-11-26T09:56:04,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964077 2024-11-26T09:56:04,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964077) roll requested 2024-11-26T09:56:04,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964084 2024-11-26T09:56:04,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964084 2024-11-26T09:56:04,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964084) roll requested 2024-11-26T09:56:04,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964091 2024-11-26T09:56:04,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964091 2024-11-26T09:56:04,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964091) roll requested 2024-11-26T09:56:04,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964099 2024-11-26T09:56:04,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964099 2024-11-26T09:56:04,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964099) roll requested 2024-11-26T09:56:04,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964107 2024-11-26T09:56:04,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964107 2024-11-26T09:56:04,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964107) roll requested 2024-11-26T09:56:04,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964116 2024-11-26T09:56:04,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964116 2024-11-26T09:56:04,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964116) roll requested 2024-11-26T09:56:04,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964124 2024-11-26T09:56:04,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964124 2024-11-26T09:56:04,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964124) roll requested 2024-11-26T09:56:04,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964131 2024-11-26T09:56:04,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964131 2024-11-26T09:56:04,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964131) roll requested 2024-11-26T09:56:04,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964138 2024-11-26T09:56:04,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964138 2024-11-26T09:56:04,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964138) roll requested 2024-11-26T09:56:04,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964146 2024-11-26T09:56:04,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964146 2024-11-26T09:56:04,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964146) roll requested 2024-11-26T09:56:04,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964154 2024-11-26T09:56:04,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964154 2024-11-26T09:56:04,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964154) roll requested 2024-11-26T09:56:04,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964162 2024-11-26T09:56:04,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964162 2024-11-26T09:56:04,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964162) roll requested 2024-11-26T09:56:04,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964170 2024-11-26T09:56:04,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964170 2024-11-26T09:56:04,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964170) roll requested 2024-11-26T09:56:04,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964177 2024-11-26T09:56:04,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964177 2024-11-26T09:56:04,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964177) roll requested 2024-11-26T09:56:04,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964184 2024-11-26T09:56:04,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964184 2024-11-26T09:56:04,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964184) roll requested 2024-11-26T09:56:04,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964191 2024-11-26T09:56:04,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964191 2024-11-26T09:56:04,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964191) roll requested 2024-11-26T09:56:04,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964198 2024-11-26T09:56:04,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964198 2024-11-26T09:56:04,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964198) roll requested 2024-11-26T09:56:04,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964212 2024-11-26T09:56:04,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964212 2024-11-26T09:56:04,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964212) roll requested 2024-11-26T09:56:04,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964220 2024-11-26T09:56:04,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964220 2024-11-26T09:56:04,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964220) roll requested 2024-11-26T09:56:04,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964228 2024-11-26T09:56:04,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964228 2024-11-26T09:56:04,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964228) roll requested 2024-11-26T09:56:04,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964236 2024-11-26T09:56:04,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964236 2024-11-26T09:56:04,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964236) roll requested 2024-11-26T09:56:04,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964243 2024-11-26T09:56:04,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964243 2024-11-26T09:56:04,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964243) roll requested 2024-11-26T09:56:04,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964252 2024-11-26T09:56:04,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964252 2024-11-26T09:56:04,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964252) roll requested 2024-11-26T09:56:04,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964263 2024-11-26T09:56:04,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964263 2024-11-26T09:56:04,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964263) roll requested 2024-11-26T09:56:04,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964271 2024-11-26T09:56:04,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964271 2024-11-26T09:56:04,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964271) roll requested 2024-11-26T09:56:04,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964279 2024-11-26T09:56:04,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964279 2024-11-26T09:56:04,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964279) roll requested 2024-11-26T09:56:04,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964287 2024-11-26T09:56:04,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964287 2024-11-26T09:56:04,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964287) roll requested 2024-11-26T09:56:04,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964294 2024-11-26T09:56:04,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964294 2024-11-26T09:56:04,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964294) roll requested 2024-11-26T09:56:04,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964302 2024-11-26T09:56:04,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964302 2024-11-26T09:56:04,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964302) roll requested 2024-11-26T09:56:04,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964310 2024-11-26T09:56:04,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964310 2024-11-26T09:56:04,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964310) roll requested 2024-11-26T09:56:04,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964318 2024-11-26T09:56:04,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964318 2024-11-26T09:56:04,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964318) roll requested 2024-11-26T09:56:04,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964325 2024-11-26T09:56:04,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964325 2024-11-26T09:56:04,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964325) roll requested 2024-11-26T09:56:04,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964333 2024-11-26T09:56:04,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964333 2024-11-26T09:56:04,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964333) roll requested 2024-11-26T09:56:04,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964340 2024-11-26T09:56:04,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964340 2024-11-26T09:56:04,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964340) roll requested 2024-11-26T09:56:04,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964347 2024-11-26T09:56:04,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964347 2024-11-26T09:56:04,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964347) roll requested 2024-11-26T09:56:04,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964353 2024-11-26T09:56:04,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964353 2024-11-26T09:56:04,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964353) roll requested 2024-11-26T09:56:04,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964360 2024-11-26T09:56:04,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964360 2024-11-26T09:56:04,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964360) roll requested 2024-11-26T09:56:04,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964368 2024-11-26T09:56:04,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964368 2024-11-26T09:56:04,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964368) roll requested 2024-11-26T09:56:04,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964375 2024-11-26T09:56:04,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964375 2024-11-26T09:56:04,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964375) roll requested 2024-11-26T09:56:04,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964382 2024-11-26T09:56:04,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964382 2024-11-26T09:56:04,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964382) roll requested 2024-11-26T09:56:04,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964389 2024-11-26T09:56:04,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964389 2024-11-26T09:56:04,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964389) roll requested 2024-11-26T09:56:04,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964396 2024-11-26T09:56:04,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964396 2024-11-26T09:56:04,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964396) roll requested 2024-11-26T09:56:04,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964403 2024-11-26T09:56:04,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964403 2024-11-26T09:56:04,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964403) roll requested 2024-11-26T09:56:04,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964410 2024-11-26T09:56:04,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964410 2024-11-26T09:56:04,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964410) roll requested 2024-11-26T09:56:04,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964417 2024-11-26T09:56:04,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964417 2024-11-26T09:56:04,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964417) roll requested 2024-11-26T09:56:04,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964425 2024-11-26T09:56:04,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964425 2024-11-26T09:56:04,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964425) roll requested 2024-11-26T09:56:04,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964433 2024-11-26T09:56:04,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964433 2024-11-26T09:56:04,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964433) roll requested 2024-11-26T09:56:04,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964440 2024-11-26T09:56:04,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964440 2024-11-26T09:56:04,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964440) roll requested 2024-11-26T09:56:04,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964448 2024-11-26T09:56:04,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964448 2024-11-26T09:56:04,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964448) roll requested 2024-11-26T09:56:04,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964466 2024-11-26T09:56:04,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964466 2024-11-26T09:56:04,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964466) roll requested 2024-11-26T09:56:04,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964473 2024-11-26T09:56:04,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964473 2024-11-26T09:56:04,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964473) roll requested 2024-11-26T09:56:04,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964481 2024-11-26T09:56:04,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964481 2024-11-26T09:56:04,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964481) roll requested 2024-11-26T09:56:04,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964489 2024-11-26T09:56:04,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964489 2024-11-26T09:56:04,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964489) roll requested 2024-11-26T09:56:04,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964496 2024-11-26T09:56:04,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964496 2024-11-26T09:56:04,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964496) roll requested 2024-11-26T09:56:04,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964504 2024-11-26T09:56:04,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964504 2024-11-26T09:56:04,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964504) roll requested 2024-11-26T09:56:04,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964512 2024-11-26T09:56:04,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964512 2024-11-26T09:56:04,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964512) roll requested 2024-11-26T09:56:04,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964519 2024-11-26T09:56:04,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964519 2024-11-26T09:56:04,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964519) roll requested 2024-11-26T09:56:04,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964527 2024-11-26T09:56:04,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964527 2024-11-26T09:56:04,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964527) roll requested 2024-11-26T09:56:04,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964534 2024-11-26T09:56:04,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964534 2024-11-26T09:56:04,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964534) roll requested 2024-11-26T09:56:04,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964542 2024-11-26T09:56:04,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964542 2024-11-26T09:56:04,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964542) roll requested 2024-11-26T09:56:04,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964549 2024-11-26T09:56:04,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964549 2024-11-26T09:56:04,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964549) roll requested 2024-11-26T09:56:04,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964556 2024-11-26T09:56:04,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964556 2024-11-26T09:56:04,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964556) roll requested 2024-11-26T09:56:04,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964564 2024-11-26T09:56:04,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964564 2024-11-26T09:56:04,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964564) roll requested 2024-11-26T09:56:04,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964572 2024-11-26T09:56:04,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964572 2024-11-26T09:56:04,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964572) roll requested 2024-11-26T09:56:04,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964579 2024-11-26T09:56:04,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964579 2024-11-26T09:56:04,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964579) roll requested 2024-11-26T09:56:04,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964592 2024-11-26T09:56:04,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964592 2024-11-26T09:56:04,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964592) roll requested 2024-11-26T09:56:04,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964599 2024-11-26T09:56:04,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964599 2024-11-26T09:56:04,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964599) roll requested 2024-11-26T09:56:04,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964606 2024-11-26T09:56:04,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964606 2024-11-26T09:56:04,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964606) roll requested 2024-11-26T09:56:04,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964613 2024-11-26T09:56:04,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964613 2024-11-26T09:56:04,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964613) roll requested 2024-11-26T09:56:04,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964620 2024-11-26T09:56:04,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964620 2024-11-26T09:56:04,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964620) roll requested 2024-11-26T09:56:04,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964628 2024-11-26T09:56:04,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964628 2024-11-26T09:56:04,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964628) roll requested 2024-11-26T09:56:04,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964635 2024-11-26T09:56:04,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964635 2024-11-26T09:56:04,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964635) roll requested 2024-11-26T09:56:04,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964643 2024-11-26T09:56:04,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964643 2024-11-26T09:56:04,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964643) roll requested 2024-11-26T09:56:04,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964650 2024-11-26T09:56:04,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964650 2024-11-26T09:56:04,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964650) roll requested 2024-11-26T09:56:04,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964657 2024-11-26T09:56:04,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964657 2024-11-26T09:56:04,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964657) roll requested 2024-11-26T09:56:04,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964664 2024-11-26T09:56:04,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964664 2024-11-26T09:56:04,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964664) roll requested 2024-11-26T09:56:04,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964671 2024-11-26T09:56:04,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964671 2024-11-26T09:56:04,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964671) roll requested 2024-11-26T09:56:04,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964678 2024-11-26T09:56:04,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964678 2024-11-26T09:56:04,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964678) roll requested 2024-11-26T09:56:04,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964685 2024-11-26T09:56:04,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964685 2024-11-26T09:56:04,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964685) roll requested 2024-11-26T09:56:04,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964692 2024-11-26T09:56:04,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964692 2024-11-26T09:56:04,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964692) roll requested 2024-11-26T09:56:04,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964699 2024-11-26T09:56:04,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964699 2024-11-26T09:56:04,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964699) roll requested 2024-11-26T09:56:04,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964706 2024-11-26T09:56:04,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964706 2024-11-26T09:56:04,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964706) roll requested 2024-11-26T09:56:04,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964714 2024-11-26T09:56:04,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964714 2024-11-26T09:56:04,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964714) roll requested 2024-11-26T09:56:04,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964721 2024-11-26T09:56:04,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964721 2024-11-26T09:56:04,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964721) roll requested 2024-11-26T09:56:04,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964728 2024-11-26T09:56:04,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964728 2024-11-26T09:56:04,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964728) roll requested 2024-11-26T09:56:04,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964735 2024-11-26T09:56:04,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964735 2024-11-26T09:56:04,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964735) roll requested 2024-11-26T09:56:04,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964742 2024-11-26T09:56:04,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964742 2024-11-26T09:56:04,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964742) roll requested 2024-11-26T09:56:04,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964750 2024-11-26T09:56:04,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964750 2024-11-26T09:56:04,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964750) roll requested 2024-11-26T09:56:04,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964757 2024-11-26T09:56:04,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964757 2024-11-26T09:56:04,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964757) roll requested 2024-11-26T09:56:04,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964768 2024-11-26T09:56:04,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964768 2024-11-26T09:56:04,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964768) roll requested 2024-11-26T09:56:04,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964776 2024-11-26T09:56:04,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964776 2024-11-26T09:56:04,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964776) roll requested 2024-11-26T09:56:04,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964783 2024-11-26T09:56:04,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964783 2024-11-26T09:56:04,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964783) roll requested 2024-11-26T09:56:04,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964795 2024-11-26T09:56:04,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964795 2024-11-26T09:56:04,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964795) roll requested 2024-11-26T09:56:04,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964802 2024-11-26T09:56:04,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964802 2024-11-26T09:56:04,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964802) roll requested 2024-11-26T09:56:04,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964808 2024-11-26T09:56:04,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964808 2024-11-26T09:56:04,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964808) roll requested 2024-11-26T09:56:04,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964815 2024-11-26T09:56:04,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964815 2024-11-26T09:56:04,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964815) roll requested 2024-11-26T09:56:04,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964822 2024-11-26T09:56:04,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964822 2024-11-26T09:56:04,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964822) roll requested 2024-11-26T09:56:04,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964829 2024-11-26T09:56:04,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964829 2024-11-26T09:56:04,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964829) roll requested 2024-11-26T09:56:04,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964836 2024-11-26T09:56:04,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964836 2024-11-26T09:56:04,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964836) roll requested 2024-11-26T09:56:04,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964843 2024-11-26T09:56:04,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964843 2024-11-26T09:56:04,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964843) roll requested 2024-11-26T09:56:04,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964850 2024-11-26T09:56:04,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964850 2024-11-26T09:56:04,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964850) roll requested 2024-11-26T09:56:04,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964857 2024-11-26T09:56:04,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964857 2024-11-26T09:56:04,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964857) roll requested 2024-11-26T09:56:04,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964864 2024-11-26T09:56:04,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964864 2024-11-26T09:56:04,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964864) roll requested 2024-11-26T09:56:04,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964871 2024-11-26T09:56:04,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964871 2024-11-26T09:56:04,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964871) roll requested 2024-11-26T09:56:04,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964878 2024-11-26T09:56:04,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964878 2024-11-26T09:56:04,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964878) roll requested 2024-11-26T09:56:04,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964886 2024-11-26T09:56:04,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964886 2024-11-26T09:56:04,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964886) roll requested 2024-11-26T09:56:04,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964893 2024-11-26T09:56:04,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964893 2024-11-26T09:56:04,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964893) roll requested 2024-11-26T09:56:04,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964900 2024-11-26T09:56:04,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964900 2024-11-26T09:56:04,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964900) roll requested 2024-11-26T09:56:04,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964907 2024-11-26T09:56:04,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964907 2024-11-26T09:56:04,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964907) roll requested 2024-11-26T09:56:04,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964914 2024-11-26T09:56:04,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964914 2024-11-26T09:56:04,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964914) roll requested 2024-11-26T09:56:04,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964924 2024-11-26T09:56:04,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964924 2024-11-26T09:56:04,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964924) roll requested 2024-11-26T09:56:04,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964935 2024-11-26T09:56:04,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964935 2024-11-26T09:56:04,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964935) roll requested 2024-11-26T09:56:04,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964956 2024-11-26T09:56:04,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964956 2024-11-26T09:56:04,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964956) roll requested 2024-11-26T09:56:04,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964964 2024-11-26T09:56:04,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964964 2024-11-26T09:56:04,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964964) roll requested 2024-11-26T09:56:04,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964971 2024-11-26T09:56:04,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964971 2024-11-26T09:56:04,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964971) roll requested 2024-11-26T09:56:04,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964978 2024-11-26T09:56:04,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964978 2024-11-26T09:56:04,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964978) roll requested 2024-11-26T09:56:04,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964991 2024-11-26T09:56:04,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:04,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964991 2024-11-26T09:56:04,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:04,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:04,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964991) roll requested 2024-11-26T09:56:04,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614964999 2024-11-26T09:56:05,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964999 2024-11-26T09:56:05,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614964999) roll requested 2024-11-26T09:56:05,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965006 2024-11-26T09:56:05,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614964999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965006 2024-11-26T09:56:05,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965006) roll requested 2024-11-26T09:56:05,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965013 2024-11-26T09:56:05,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965013 2024-11-26T09:56:05,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965013) roll requested 2024-11-26T09:56:05,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965020 2024-11-26T09:56:05,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965020 2024-11-26T09:56:05,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965020) roll requested 2024-11-26T09:56:05,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965027 2024-11-26T09:56:05,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965027 2024-11-26T09:56:05,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965027) roll requested 2024-11-26T09:56:05,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965034 2024-11-26T09:56:05,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965034 2024-11-26T09:56:05,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965034) roll requested 2024-11-26T09:56:05,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965041 2024-11-26T09:56:05,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965041 2024-11-26T09:56:05,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965041) roll requested 2024-11-26T09:56:05,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965048 2024-11-26T09:56:05,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965048 2024-11-26T09:56:05,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965048) roll requested 2024-11-26T09:56:05,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965055 2024-11-26T09:56:05,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965055 2024-11-26T09:56:05,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965055) roll requested 2024-11-26T09:56:05,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965062 2024-11-26T09:56:05,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965062 2024-11-26T09:56:05,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965062) roll requested 2024-11-26T09:56:05,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965069 2024-11-26T09:56:05,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965069 2024-11-26T09:56:05,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965069) roll requested 2024-11-26T09:56:05,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965076 2024-11-26T09:56:05,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965076 2024-11-26T09:56:05,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965076) roll requested 2024-11-26T09:56:05,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965083 2024-11-26T09:56:05,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965083 2024-11-26T09:56:05,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965083) roll requested 2024-11-26T09:56:05,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965091 2024-11-26T09:56:05,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965091 2024-11-26T09:56:05,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965091) roll requested 2024-11-26T09:56:05,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965098 2024-11-26T09:56:05,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965098 2024-11-26T09:56:05,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965098) roll requested 2024-11-26T09:56:05,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965105 2024-11-26T09:56:05,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965105 2024-11-26T09:56:05,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965105) roll requested 2024-11-26T09:56:05,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965113 2024-11-26T09:56:05,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965113 2024-11-26T09:56:05,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965113) roll requested 2024-11-26T09:56:05,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965120 2024-11-26T09:56:05,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965120 2024-11-26T09:56:05,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965120) roll requested 2024-11-26T09:56:05,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965128 2024-11-26T09:56:05,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965128 2024-11-26T09:56:05,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965128) roll requested 2024-11-26T09:56:05,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965134 2024-11-26T09:56:05,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965134 2024-11-26T09:56:05,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965134) roll requested 2024-11-26T09:56:05,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965142 2024-11-26T09:56:05,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965142 2024-11-26T09:56:05,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965142) roll requested 2024-11-26T09:56:05,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965148 2024-11-26T09:56:05,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965148 2024-11-26T09:56:05,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965148) roll requested 2024-11-26T09:56:05,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965156 2024-11-26T09:56:05,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965156 2024-11-26T09:56:05,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965156) roll requested 2024-11-26T09:56:05,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965163 2024-11-26T09:56:05,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965163 2024-11-26T09:56:05,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965163) roll requested 2024-11-26T09:56:05,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965170 2024-11-26T09:56:05,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965170 2024-11-26T09:56:05,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965170) roll requested 2024-11-26T09:56:05,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965177 2024-11-26T09:56:05,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965177 2024-11-26T09:56:05,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965177) roll requested 2024-11-26T09:56:05,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965185 2024-11-26T09:56:05,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965185 2024-11-26T09:56:05,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965185) roll requested 2024-11-26T09:56:05,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965192 2024-11-26T09:56:05,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965192 2024-11-26T09:56:05,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965192) roll requested 2024-11-26T09:56:05,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965205 2024-11-26T09:56:05,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965205 2024-11-26T09:56:05,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965205) roll requested 2024-11-26T09:56:05,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965215 2024-11-26T09:56:05,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965215 2024-11-26T09:56:05,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965215) roll requested 2024-11-26T09:56:05,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965222 2024-11-26T09:56:05,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965222 2024-11-26T09:56:05,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965222) roll requested 2024-11-26T09:56:05,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965230 2024-11-26T09:56:05,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965230 2024-11-26T09:56:05,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965230) roll requested 2024-11-26T09:56:05,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965237 2024-11-26T09:56:05,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965237 2024-11-26T09:56:05,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965237) roll requested 2024-11-26T09:56:05,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965244 2024-11-26T09:56:05,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965244 2024-11-26T09:56:05,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965244) roll requested 2024-11-26T09:56:05,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965252 2024-11-26T09:56:05,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965252 2024-11-26T09:56:05,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965252) roll requested 2024-11-26T09:56:05,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965258 2024-11-26T09:56:05,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965258 2024-11-26T09:56:05,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965258) roll requested 2024-11-26T09:56:05,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965266 2024-11-26T09:56:05,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965266 2024-11-26T09:56:05,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965266) roll requested 2024-11-26T09:56:05,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965273 2024-11-26T09:56:05,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965273 2024-11-26T09:56:05,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965273) roll requested 2024-11-26T09:56:05,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965280 2024-11-26T09:56:05,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965280 2024-11-26T09:56:05,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965280) roll requested 2024-11-26T09:56:05,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965288 2024-11-26T09:56:05,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965288 2024-11-26T09:56:05,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965288) roll requested 2024-11-26T09:56:05,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965296 2024-11-26T09:56:05,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965296 2024-11-26T09:56:05,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965296) roll requested 2024-11-26T09:56:05,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965304 2024-11-26T09:56:05,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965304 2024-11-26T09:56:05,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965304) roll requested 2024-11-26T09:56:05,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965310 2024-11-26T09:56:05,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965310 2024-11-26T09:56:05,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965310) roll requested 2024-11-26T09:56:05,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965317 2024-11-26T09:56:05,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965317 2024-11-26T09:56:05,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965317) roll requested 2024-11-26T09:56:05,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965324 2024-11-26T09:56:05,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965324 2024-11-26T09:56:05,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965324) roll requested 2024-11-26T09:56:05,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965332 2024-11-26T09:56:05,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965332 2024-11-26T09:56:05,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965332) roll requested 2024-11-26T09:56:05,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965339 2024-11-26T09:56:05,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965339 2024-11-26T09:56:05,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965339) roll requested 2024-11-26T09:56:05,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965346 2024-11-26T09:56:05,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965346 2024-11-26T09:56:05,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965346) roll requested 2024-11-26T09:56:05,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965354 2024-11-26T09:56:05,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965354 2024-11-26T09:56:05,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965354) roll requested 2024-11-26T09:56:05,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965365 2024-11-26T09:56:05,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965365 2024-11-26T09:56:05,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965365) roll requested 2024-11-26T09:56:05,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965372 2024-11-26T09:56:05,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965372 2024-11-26T09:56:05,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965372) roll requested 2024-11-26T09:56:05,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965379 2024-11-26T09:56:05,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965379 2024-11-26T09:56:05,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965379) roll requested 2024-11-26T09:56:05,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965386 2024-11-26T09:56:05,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965386 2024-11-26T09:56:05,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965386) roll requested 2024-11-26T09:56:05,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965394 2024-11-26T09:56:05,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965394 2024-11-26T09:56:05,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965394) roll requested 2024-11-26T09:56:05,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965401 2024-11-26T09:56:05,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965401 2024-11-26T09:56:05,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965401) roll requested 2024-11-26T09:56:05,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965408 2024-11-26T09:56:05,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965408 2024-11-26T09:56:05,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965408) roll requested 2024-11-26T09:56:05,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965415 2024-11-26T09:56:05,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965415 2024-11-26T09:56:05,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965415) roll requested 2024-11-26T09:56:05,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965422 2024-11-26T09:56:05,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965422 2024-11-26T09:56:05,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965422) roll requested 2024-11-26T09:56:05,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965429 2024-11-26T09:56:05,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965429 2024-11-26T09:56:05,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965429) roll requested 2024-11-26T09:56:05,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965436 2024-11-26T09:56:05,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965436 2024-11-26T09:56:05,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965436) roll requested 2024-11-26T09:56:05,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965443 2024-11-26T09:56:05,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965443 2024-11-26T09:56:05,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965443) roll requested 2024-11-26T09:56:05,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965450 2024-11-26T09:56:05,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965450 2024-11-26T09:56:05,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965450) roll requested 2024-11-26T09:56:05,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965464 2024-11-26T09:56:05,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965464 2024-11-26T09:56:05,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965464) roll requested 2024-11-26T09:56:05,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965473 2024-11-26T09:56:05,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965473 2024-11-26T09:56:05,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965473) roll requested 2024-11-26T09:56:05,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965480 2024-11-26T09:56:05,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965480 2024-11-26T09:56:05,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965480) roll requested 2024-11-26T09:56:05,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965488 2024-11-26T09:56:05,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965488 2024-11-26T09:56:05,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965488) roll requested 2024-11-26T09:56:05,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965501 2024-11-26T09:56:05,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965501 2024-11-26T09:56:05,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965501) roll requested 2024-11-26T09:56:05,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965508 2024-11-26T09:56:05,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965508 2024-11-26T09:56:05,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965508) roll requested 2024-11-26T09:56:05,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965515 2024-11-26T09:56:05,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965515 2024-11-26T09:56:05,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965515) roll requested 2024-11-26T09:56:05,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965522 2024-11-26T09:56:05,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965522 2024-11-26T09:56:05,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965522) roll requested 2024-11-26T09:56:05,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965530 2024-11-26T09:56:05,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965530 2024-11-26T09:56:05,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965530) roll requested 2024-11-26T09:56:05,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965537 2024-11-26T09:56:05,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965537 2024-11-26T09:56:05,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965537) roll requested 2024-11-26T09:56:05,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965543 2024-11-26T09:56:05,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965543 2024-11-26T09:56:05,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965543) roll requested 2024-11-26T09:56:05,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965550 2024-11-26T09:56:05,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965550 2024-11-26T09:56:05,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965550) roll requested 2024-11-26T09:56:05,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965558 2024-11-26T09:56:05,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965558 2024-11-26T09:56:05,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965558) roll requested 2024-11-26T09:56:05,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965565 2024-11-26T09:56:05,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965565 2024-11-26T09:56:05,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965565) roll requested 2024-11-26T09:56:05,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965572 2024-11-26T09:56:05,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965572 2024-11-26T09:56:05,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965572) roll requested 2024-11-26T09:56:05,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965580 2024-11-26T09:56:05,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965580 2024-11-26T09:56:05,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965580) roll requested 2024-11-26T09:56:05,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965587 2024-11-26T09:56:05,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965587 2024-11-26T09:56:05,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965587) roll requested 2024-11-26T09:56:05,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965594 2024-11-26T09:56:05,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965594 2024-11-26T09:56:05,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965594) roll requested 2024-11-26T09:56:05,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965601 2024-11-26T09:56:05,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965601 2024-11-26T09:56:05,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965601) roll requested 2024-11-26T09:56:05,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965608 2024-11-26T09:56:05,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965608 2024-11-26T09:56:05,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965608) roll requested 2024-11-26T09:56:05,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965614 2024-11-26T09:56:05,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965614 2024-11-26T09:56:05,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965614) roll requested 2024-11-26T09:56:05,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965621 2024-11-26T09:56:05,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965621 2024-11-26T09:56:05,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965621) roll requested 2024-11-26T09:56:05,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965628 2024-11-26T09:56:05,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965628 2024-11-26T09:56:05,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965628) roll requested 2024-11-26T09:56:05,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965635 2024-11-26T09:56:05,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965635 2024-11-26T09:56:05,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965635) roll requested 2024-11-26T09:56:05,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965642 2024-11-26T09:56:05,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965642 2024-11-26T09:56:05,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965642) roll requested 2024-11-26T09:56:05,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965649 2024-11-26T09:56:05,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965649 2024-11-26T09:56:05,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965649) roll requested 2024-11-26T09:56:05,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965655 2024-11-26T09:56:05,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965655 2024-11-26T09:56:05,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965655) roll requested 2024-11-26T09:56:05,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965662 2024-11-26T09:56:05,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965662 2024-11-26T09:56:05,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965662) roll requested 2024-11-26T09:56:05,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965669 2024-11-26T09:56:05,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965669 2024-11-26T09:56:05,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965669) roll requested 2024-11-26T09:56:05,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965680 2024-11-26T09:56:05,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965680 2024-11-26T09:56:05,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965680) roll requested 2024-11-26T09:56:05,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965687 2024-11-26T09:56:05,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965687 2024-11-26T09:56:05,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965687) roll requested 2024-11-26T09:56:05,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965694 2024-11-26T09:56:05,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965694 2024-11-26T09:56:05,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965694) roll requested 2024-11-26T09:56:05,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965701 2024-11-26T09:56:05,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965701 2024-11-26T09:56:05,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965701) roll requested 2024-11-26T09:56:05,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965708 2024-11-26T09:56:05,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965708 2024-11-26T09:56:05,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965708) roll requested 2024-11-26T09:56:05,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965715 2024-11-26T09:56:05,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965715 2024-11-26T09:56:05,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965715) roll requested 2024-11-26T09:56:05,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965721 2024-11-26T09:56:05,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965721 2024-11-26T09:56:05,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965721) roll requested 2024-11-26T09:56:05,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965728 2024-11-26T09:56:05,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965728 2024-11-26T09:56:05,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965728) roll requested 2024-11-26T09:56:05,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965735 2024-11-26T09:56:05,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965735 2024-11-26T09:56:05,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965735) roll requested 2024-11-26T09:56:05,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965742 2024-11-26T09:56:05,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965742 2024-11-26T09:56:05,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965742) roll requested 2024-11-26T09:56:05,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965749 2024-11-26T09:56:05,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965749 2024-11-26T09:56:05,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965749) roll requested 2024-11-26T09:56:05,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965757 2024-11-26T09:56:05,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965757 2024-11-26T09:56:05,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965757) roll requested 2024-11-26T09:56:05,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965764 2024-11-26T09:56:05,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965764 2024-11-26T09:56:05,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965764) roll requested 2024-11-26T09:56:05,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965771 2024-11-26T09:56:05,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965771 2024-11-26T09:56:05,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965771) roll requested 2024-11-26T09:56:05,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965778 2024-11-26T09:56:05,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965778 2024-11-26T09:56:05,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965778) roll requested 2024-11-26T09:56:05,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965786 2024-11-26T09:56:05,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965786 2024-11-26T09:56:05,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965786) roll requested 2024-11-26T09:56:05,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965793 2024-11-26T09:56:05,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965793 2024-11-26T09:56:05,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965793) roll requested 2024-11-26T09:56:05,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965800 2024-11-26T09:56:05,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965800 2024-11-26T09:56:05,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965800) roll requested 2024-11-26T09:56:05,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965807 2024-11-26T09:56:05,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965807 2024-11-26T09:56:05,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965807) roll requested 2024-11-26T09:56:05,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965814 2024-11-26T09:56:05,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965814 2024-11-26T09:56:05,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965814) roll requested 2024-11-26T09:56:05,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965821 2024-11-26T09:56:05,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965821 2024-11-26T09:56:05,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965821) roll requested 2024-11-26T09:56:05,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965829 2024-11-26T09:56:05,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965829 2024-11-26T09:56:05,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965829) roll requested 2024-11-26T09:56:05,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965835 2024-11-26T09:56:05,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965835 2024-11-26T09:56:05,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965835) roll requested 2024-11-26T09:56:05,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965846 2024-11-26T09:56:05,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965846 2024-11-26T09:56:05,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965846) roll requested 2024-11-26T09:56:05,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965853 2024-11-26T09:56:05,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965853 2024-11-26T09:56:05,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965853) roll requested 2024-11-26T09:56:05,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965860 2024-11-26T09:56:05,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965860 2024-11-26T09:56:05,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965860) roll requested 2024-11-26T09:56:05,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965867 2024-11-26T09:56:05,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965867 2024-11-26T09:56:05,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965867) roll requested 2024-11-26T09:56:05,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965874 2024-11-26T09:56:05,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965874 2024-11-26T09:56:05,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965874) roll requested 2024-11-26T09:56:05,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965882 2024-11-26T09:56:05,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965882 2024-11-26T09:56:05,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965882) roll requested 2024-11-26T09:56:05,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965889 2024-11-26T09:56:05,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965889 2024-11-26T09:56:05,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965889) roll requested 2024-11-26T09:56:05,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965895 2024-11-26T09:56:05,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965895 2024-11-26T09:56:05,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965895) roll requested 2024-11-26T09:56:05,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965902 2024-11-26T09:56:05,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965902 2024-11-26T09:56:05,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965902) roll requested 2024-11-26T09:56:05,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965909 2024-11-26T09:56:05,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965909 2024-11-26T09:56:05,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965909) roll requested 2024-11-26T09:56:05,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965915 2024-11-26T09:56:05,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965915 2024-11-26T09:56:05,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965915) roll requested 2024-11-26T09:56:05,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965922 2024-11-26T09:56:05,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965922 2024-11-26T09:56:05,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965922) roll requested 2024-11-26T09:56:05,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965928 2024-11-26T09:56:05,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965928 2024-11-26T09:56:05,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965928) roll requested 2024-11-26T09:56:05,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965935 2024-11-26T09:56:05,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965935 2024-11-26T09:56:05,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965935) roll requested 2024-11-26T09:56:05,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965942 2024-11-26T09:56:05,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965942 2024-11-26T09:56:05,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965942) roll requested 2024-11-26T09:56:05,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965949 2024-11-26T09:56:05,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965949 2024-11-26T09:56:05,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965949) roll requested 2024-11-26T09:56:05,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965956 2024-11-26T09:56:05,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965956 2024-11-26T09:56:05,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965956) roll requested 2024-11-26T09:56:05,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965963 2024-11-26T09:56:05,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965963 2024-11-26T09:56:05,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965963) roll requested 2024-11-26T09:56:05,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965971 2024-11-26T09:56:05,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965971 2024-11-26T09:56:05,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965971) roll requested 2024-11-26T09:56:05,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965978 2024-11-26T09:56:05,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965978 2024-11-26T09:56:05,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965978) roll requested 2024-11-26T09:56:05,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965986 2024-11-26T09:56:05,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965986 2024-11-26T09:56:05,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965986) roll requested 2024-11-26T09:56:05,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614965992 2024-11-26T09:56:05,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:05,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965992 2024-11-26T09:56:05,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:05,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:05,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614965992) roll requested 2024-11-26T09:56:06,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966000 2024-11-26T09:56:06,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614965992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966000 2024-11-26T09:56:06,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966000) roll requested 2024-11-26T09:56:06,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966007 2024-11-26T09:56:06,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966007 2024-11-26T09:56:06,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966007) roll requested 2024-11-26T09:56:06,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966013 2024-11-26T09:56:06,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966013 2024-11-26T09:56:06,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966013) roll requested 2024-11-26T09:56:06,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966020 2024-11-26T09:56:06,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966020 2024-11-26T09:56:06,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966020) roll requested 2024-11-26T09:56:06,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966027 2024-11-26T09:56:06,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966027 2024-11-26T09:56:06,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966027) roll requested 2024-11-26T09:56:06,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966035 2024-11-26T09:56:06,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966035 2024-11-26T09:56:06,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966035) roll requested 2024-11-26T09:56:06,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966041 2024-11-26T09:56:06,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966041 2024-11-26T09:56:06,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966041) roll requested 2024-11-26T09:56:06,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966048 2024-11-26T09:56:06,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966048 2024-11-26T09:56:06,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966048) roll requested 2024-11-26T09:56:06,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966058 2024-11-26T09:56:06,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966058 2024-11-26T09:56:06,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966058) roll requested 2024-11-26T09:56:06,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966065 2024-11-26T09:56:06,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966065 2024-11-26T09:56:06,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966065) roll requested 2024-11-26T09:56:06,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966072 2024-11-26T09:56:06,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966072 2024-11-26T09:56:06,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966072) roll requested 2024-11-26T09:56:06,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966079 2024-11-26T09:56:06,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966079 2024-11-26T09:56:06,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966079) roll requested 2024-11-26T09:56:06,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966086 2024-11-26T09:56:06,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966086 2024-11-26T09:56:06,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966086) roll requested 2024-11-26T09:56:06,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966093 2024-11-26T09:56:06,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966093 2024-11-26T09:56:06,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966093) roll requested 2024-11-26T09:56:06,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966100 2024-11-26T09:56:06,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966100 2024-11-26T09:56:06,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966100) roll requested 2024-11-26T09:56:06,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966107 2024-11-26T09:56:06,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966107 2024-11-26T09:56:06,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966107) roll requested 2024-11-26T09:56:06,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966113 2024-11-26T09:56:06,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966113 2024-11-26T09:56:06,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966113) roll requested 2024-11-26T09:56:06,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966128 2024-11-26T09:56:06,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966128 2024-11-26T09:56:06,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966128) roll requested 2024-11-26T09:56:06,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966140 2024-11-26T09:56:06,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966140 2024-11-26T09:56:06,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966140) roll requested 2024-11-26T09:56:06,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966150 2024-11-26T09:56:06,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966150 2024-11-26T09:56:06,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966150) roll requested 2024-11-26T09:56:06,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966158 2024-11-26T09:56:06,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966158 2024-11-26T09:56:06,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966158) roll requested 2024-11-26T09:56:06,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966165 2024-11-26T09:56:06,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966165 2024-11-26T09:56:06,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966165) roll requested 2024-11-26T09:56:06,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966172 2024-11-26T09:56:06,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966172 2024-11-26T09:56:06,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966172) roll requested 2024-11-26T09:56:06,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966179 2024-11-26T09:56:06,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966179 2024-11-26T09:56:06,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966179) roll requested 2024-11-26T09:56:06,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966187 2024-11-26T09:56:06,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966187 2024-11-26T09:56:06,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966187) roll requested 2024-11-26T09:56:06,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966194 2024-11-26T09:56:06,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966194 2024-11-26T09:56:06,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966194) roll requested 2024-11-26T09:56:06,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966201 2024-11-26T09:56:06,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966201 2024-11-26T09:56:06,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966201) roll requested 2024-11-26T09:56:06,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966209 2024-11-26T09:56:06,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966209 2024-11-26T09:56:06,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966209) roll requested 2024-11-26T09:56:06,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966217 2024-11-26T09:56:06,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966217 2024-11-26T09:56:06,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966217) roll requested 2024-11-26T09:56:06,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966224 2024-11-26T09:56:06,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966224 2024-11-26T09:56:06,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966224) roll requested 2024-11-26T09:56:06,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966231 2024-11-26T09:56:06,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966231 2024-11-26T09:56:06,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966231) roll requested 2024-11-26T09:56:06,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966239 2024-11-26T09:56:06,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966239 2024-11-26T09:56:06,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966239) roll requested 2024-11-26T09:56:06,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966248 2024-11-26T09:56:06,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966248 2024-11-26T09:56:06,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966248) roll requested 2024-11-26T09:56:06,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966256 2024-11-26T09:56:06,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966256 2024-11-26T09:56:06,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966256) roll requested 2024-11-26T09:56:06,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966263 2024-11-26T09:56:06,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966263 2024-11-26T09:56:06,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966263) roll requested 2024-11-26T09:56:06,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966270 2024-11-26T09:56:06,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966270 2024-11-26T09:56:06,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966270) roll requested 2024-11-26T09:56:06,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966277 2024-11-26T09:56:06,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966277 2024-11-26T09:56:06,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966277) roll requested 2024-11-26T09:56:06,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966286 2024-11-26T09:56:06,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966286 2024-11-26T09:56:06,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966286) roll requested 2024-11-26T09:56:06,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966293 2024-11-26T09:56:06,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966293 2024-11-26T09:56:06,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966293) roll requested 2024-11-26T09:56:06,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966300 2024-11-26T09:56:06,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966300 2024-11-26T09:56:06,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966300) roll requested 2024-11-26T09:56:06,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966308 2024-11-26T09:56:06,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966308 2024-11-26T09:56:06,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966308) roll requested 2024-11-26T09:56:06,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966317 2024-11-26T09:56:06,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966317 2024-11-26T09:56:06,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966317) roll requested 2024-11-26T09:56:06,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966324 2024-11-26T09:56:06,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966324 2024-11-26T09:56:06,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966324) roll requested 2024-11-26T09:56:06,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966332 2024-11-26T09:56:06,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966332 2024-11-26T09:56:06,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966332) roll requested 2024-11-26T09:56:06,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966346 2024-11-26T09:56:06,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966346 2024-11-26T09:56:06,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966346) roll requested 2024-11-26T09:56:06,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966353 2024-11-26T09:56:06,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966353 2024-11-26T09:56:06,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966353) roll requested 2024-11-26T09:56:06,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966365 2024-11-26T09:56:06,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966365 2024-11-26T09:56:06,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966365) roll requested 2024-11-26T09:56:06,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966373 2024-11-26T09:56:06,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966373 2024-11-26T09:56:06,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966373) roll requested 2024-11-26T09:56:06,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966381 2024-11-26T09:56:06,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966381 2024-11-26T09:56:06,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966381) roll requested 2024-11-26T09:56:06,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966389 2024-11-26T09:56:06,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966389 2024-11-26T09:56:06,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966389) roll requested 2024-11-26T09:56:06,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966397 2024-11-26T09:56:06,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966397 2024-11-26T09:56:06,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966397) roll requested 2024-11-26T09:56:06,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966403 2024-11-26T09:56:06,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966403 2024-11-26T09:56:06,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966403) roll requested 2024-11-26T09:56:06,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966411 2024-11-26T09:56:06,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966411 2024-11-26T09:56:06,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966411) roll requested 2024-11-26T09:56:06,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966418 2024-11-26T09:56:06,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966418 2024-11-26T09:56:06,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966418) roll requested 2024-11-26T09:56:06,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966425 2024-11-26T09:56:06,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966425 2024-11-26T09:56:06,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966425) roll requested 2024-11-26T09:56:06,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966432 2024-11-26T09:56:06,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966432 2024-11-26T09:56:06,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966432) roll requested 2024-11-26T09:56:06,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966439 2024-11-26T09:56:06,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966439 2024-11-26T09:56:06,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966439) roll requested 2024-11-26T09:56:06,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966445 2024-11-26T09:56:06,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966445 2024-11-26T09:56:06,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966445) roll requested 2024-11-26T09:56:06,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966452 2024-11-26T09:56:06,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966452 2024-11-26T09:56:06,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966452) roll requested 2024-11-26T09:56:06,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966459 2024-11-26T09:56:06,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966459 2024-11-26T09:56:06,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966459) roll requested 2024-11-26T09:56:06,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966466 2024-11-26T09:56:06,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966466 2024-11-26T09:56:06,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966466) roll requested 2024-11-26T09:56:06,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966472 2024-11-26T09:56:06,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966472 2024-11-26T09:56:06,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966472) roll requested 2024-11-26T09:56:06,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966479 2024-11-26T09:56:06,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966479 2024-11-26T09:56:06,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966479) roll requested 2024-11-26T09:56:06,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966486 2024-11-26T09:56:06,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966486 2024-11-26T09:56:06,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966486) roll requested 2024-11-26T09:56:06,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966494 2024-11-26T09:56:06,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966494 2024-11-26T09:56:06,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966494) roll requested 2024-11-26T09:56:06,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966501 2024-11-26T09:56:06,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966501 2024-11-26T09:56:06,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966501) roll requested 2024-11-26T09:56:06,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966508 2024-11-26T09:56:06,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966508 2024-11-26T09:56:06,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966508) roll requested 2024-11-26T09:56:06,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966515 2024-11-26T09:56:06,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966515 2024-11-26T09:56:06,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966515) roll requested 2024-11-26T09:56:06,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966521 2024-11-26T09:56:06,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966521 2024-11-26T09:56:06,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966521) roll requested 2024-11-26T09:56:06,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966528 2024-11-26T09:56:06,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966528 2024-11-26T09:56:06,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966528) roll requested 2024-11-26T09:56:06,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966535 2024-11-26T09:56:06,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966535 2024-11-26T09:56:06,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966535) roll requested 2024-11-26T09:56:06,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966542 2024-11-26T09:56:06,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966542 2024-11-26T09:56:06,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966542) roll requested 2024-11-26T09:56:06,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966549 2024-11-26T09:56:06,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966549 2024-11-26T09:56:06,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966549) roll requested 2024-11-26T09:56:06,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966555 2024-11-26T09:56:06,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966555 2024-11-26T09:56:06,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966555) roll requested 2024-11-26T09:56:06,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966562 2024-11-26T09:56:06,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966562 2024-11-26T09:56:06,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966562) roll requested 2024-11-26T09:56:06,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966568 2024-11-26T09:56:06,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966568 2024-11-26T09:56:06,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966568) roll requested 2024-11-26T09:56:06,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966575 2024-11-26T09:56:06,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966575 2024-11-26T09:56:06,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966575) roll requested 2024-11-26T09:56:06,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966582 2024-11-26T09:56:06,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966582 2024-11-26T09:56:06,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966582) roll requested 2024-11-26T09:56:06,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966589 2024-11-26T09:56:06,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966589 2024-11-26T09:56:06,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966589) roll requested 2024-11-26T09:56:06,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966596 2024-11-26T09:56:06,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966596 2024-11-26T09:56:06,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966596) roll requested 2024-11-26T09:56:06,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966603 2024-11-26T09:56:06,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966603 2024-11-26T09:56:06,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966603) roll requested 2024-11-26T09:56:06,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966610 2024-11-26T09:56:06,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966610 2024-11-26T09:56:06,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966610) roll requested 2024-11-26T09:56:06,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966617 2024-11-26T09:56:06,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966617 2024-11-26T09:56:06,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966617) roll requested 2024-11-26T09:56:06,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966624 2024-11-26T09:56:06,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966624 2024-11-26T09:56:06,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966624) roll requested 2024-11-26T09:56:06,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966630 2024-11-26T09:56:06,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966630 2024-11-26T09:56:06,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966630) roll requested 2024-11-26T09:56:06,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966637 2024-11-26T09:56:06,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966637 2024-11-26T09:56:06,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966637) roll requested 2024-11-26T09:56:06,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966644 2024-11-26T09:56:06,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966644 2024-11-26T09:56:06,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966644) roll requested 2024-11-26T09:56:06,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966651 2024-11-26T09:56:06,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966651 2024-11-26T09:56:06,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966651) roll requested 2024-11-26T09:56:06,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966658 2024-11-26T09:56:06,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966658 2024-11-26T09:56:06,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966658) roll requested 2024-11-26T09:56:06,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966665 2024-11-26T09:56:06,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966665 2024-11-26T09:56:06,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966665) roll requested 2024-11-26T09:56:06,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966671 2024-11-26T09:56:06,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966671 2024-11-26T09:56:06,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966671) roll requested 2024-11-26T09:56:06,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966679 2024-11-26T09:56:06,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966679 2024-11-26T09:56:06,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966679) roll requested 2024-11-26T09:56:06,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966686 2024-11-26T09:56:06,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966686 2024-11-26T09:56:06,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966686) roll requested 2024-11-26T09:56:06,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966693 2024-11-26T09:56:06,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966693 2024-11-26T09:56:06,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966693) roll requested 2024-11-26T09:56:06,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966700 2024-11-26T09:56:06,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966700 2024-11-26T09:56:06,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966700) roll requested 2024-11-26T09:56:06,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966707 2024-11-26T09:56:06,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966707 2024-11-26T09:56:06,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966707) roll requested 2024-11-26T09:56:06,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966714 2024-11-26T09:56:06,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966714 2024-11-26T09:56:06,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966714) roll requested 2024-11-26T09:56:06,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966721 2024-11-26T09:56:06,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966721 2024-11-26T09:56:06,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966721) roll requested 2024-11-26T09:56:06,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966728 2024-11-26T09:56:06,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966728 2024-11-26T09:56:06,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966728) roll requested 2024-11-26T09:56:06,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966735 2024-11-26T09:56:06,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966735 2024-11-26T09:56:06,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966735) roll requested 2024-11-26T09:56:06,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966742 2024-11-26T09:56:06,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966742 2024-11-26T09:56:06,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966742) roll requested 2024-11-26T09:56:06,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966749 2024-11-26T09:56:06,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966749 2024-11-26T09:56:06,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966749) roll requested 2024-11-26T09:56:06,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966756 2024-11-26T09:56:06,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966756 2024-11-26T09:56:06,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966756) roll requested 2024-11-26T09:56:06,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966763 2024-11-26T09:56:06,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966763 2024-11-26T09:56:06,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966763) roll requested 2024-11-26T09:56:06,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966769 2024-11-26T09:56:06,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966769 2024-11-26T09:56:06,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966769) roll requested 2024-11-26T09:56:06,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966777 2024-11-26T09:56:06,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966777 2024-11-26T09:56:06,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966777) roll requested 2024-11-26T09:56:06,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966784 2024-11-26T09:56:06,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966784 2024-11-26T09:56:06,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966784) roll requested 2024-11-26T09:56:06,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966790 2024-11-26T09:56:06,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966790 2024-11-26T09:56:06,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966790) roll requested 2024-11-26T09:56:06,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966797 2024-11-26T09:56:06,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966797 2024-11-26T09:56:06,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966797) roll requested 2024-11-26T09:56:06,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966804 2024-11-26T09:56:06,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966804 2024-11-26T09:56:06,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966804) roll requested 2024-11-26T09:56:06,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966811 2024-11-26T09:56:06,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966811 2024-11-26T09:56:06,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966811) roll requested 2024-11-26T09:56:06,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966819 2024-11-26T09:56:06,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966819 2024-11-26T09:56:06,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966819) roll requested 2024-11-26T09:56:06,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966826 2024-11-26T09:56:06,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966826 2024-11-26T09:56:06,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966826) roll requested 2024-11-26T09:56:06,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966833 2024-11-26T09:56:06,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966833 2024-11-26T09:56:06,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966833) roll requested 2024-11-26T09:56:06,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966840 2024-11-26T09:56:06,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966840 2024-11-26T09:56:06,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966840) roll requested 2024-11-26T09:56:06,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966847 2024-11-26T09:56:06,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966847 2024-11-26T09:56:06,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966847) roll requested 2024-11-26T09:56:06,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966854 2024-11-26T09:56:06,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966854 2024-11-26T09:56:06,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966854) roll requested 2024-11-26T09:56:06,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966861 2024-11-26T09:56:06,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966861 2024-11-26T09:56:06,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966861) roll requested 2024-11-26T09:56:06,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966868 2024-11-26T09:56:06,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966868 2024-11-26T09:56:06,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966868) roll requested 2024-11-26T09:56:06,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966875 2024-11-26T09:56:06,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966875 2024-11-26T09:56:06,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966875) roll requested 2024-11-26T09:56:06,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966882 2024-11-26T09:56:06,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966882 2024-11-26T09:56:06,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966882) roll requested 2024-11-26T09:56:06,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966889 2024-11-26T09:56:06,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966889 2024-11-26T09:56:06,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966889) roll requested 2024-11-26T09:56:06,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966896 2024-11-26T09:56:06,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966896 2024-11-26T09:56:06,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966896) roll requested 2024-11-26T09:56:06,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966903 2024-11-26T09:56:06,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966903 2024-11-26T09:56:06,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966903) roll requested 2024-11-26T09:56:06,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966913 2024-11-26T09:56:06,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966913 2024-11-26T09:56:06,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966913) roll requested 2024-11-26T09:56:06,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966920 2024-11-26T09:56:06,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966920 2024-11-26T09:56:06,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966920) roll requested 2024-11-26T09:56:06,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966928 2024-11-26T09:56:06,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966928 2024-11-26T09:56:06,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966928) roll requested 2024-11-26T09:56:06,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966935 2024-11-26T09:56:06,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966935 2024-11-26T09:56:06,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966935) roll requested 2024-11-26T09:56:06,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966941 2024-11-26T09:56:06,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966941 2024-11-26T09:56:06,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966941) roll requested 2024-11-26T09:56:06,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966948 2024-11-26T09:56:06,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966948 2024-11-26T09:56:06,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966948) roll requested 2024-11-26T09:56:06,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966955 2024-11-26T09:56:06,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966955 2024-11-26T09:56:06,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966955) roll requested 2024-11-26T09:56:06,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966961 2024-11-26T09:56:06,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966961 2024-11-26T09:56:06,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966961) roll requested 2024-11-26T09:56:06,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966968 2024-11-26T09:56:06,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966968 2024-11-26T09:56:06,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966968) roll requested 2024-11-26T09:56:06,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966980 2024-11-26T09:56:06,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966980 2024-11-26T09:56:06,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966980) roll requested 2024-11-26T09:56:06,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966988 2024-11-26T09:56:06,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:06,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966988 2024-11-26T09:56:06,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:06,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:06,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966988) roll requested 2024-11-26T09:56:06,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614966995 2024-11-26T09:56:07,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966995 2024-11-26T09:56:07,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614966995) roll requested 2024-11-26T09:56:07,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967001 2024-11-26T09:56:07,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614966995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967001 2024-11-26T09:56:07,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967001) roll requested 2024-11-26T09:56:07,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967009 2024-11-26T09:56:07,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967009 2024-11-26T09:56:07,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967009) roll requested 2024-11-26T09:56:07,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967016 2024-11-26T09:56:07,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967016 2024-11-26T09:56:07,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967016) roll requested 2024-11-26T09:56:07,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967026 2024-11-26T09:56:07,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967026 2024-11-26T09:56:07,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967026) roll requested 2024-11-26T09:56:07,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967034 2024-11-26T09:56:07,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967034 2024-11-26T09:56:07,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967034) roll requested 2024-11-26T09:56:07,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967041 2024-11-26T09:56:07,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967041 2024-11-26T09:56:07,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967041) roll requested 2024-11-26T09:56:07,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967048 2024-11-26T09:56:07,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967048 2024-11-26T09:56:07,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967048) roll requested 2024-11-26T09:56:07,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967054 2024-11-26T09:56:07,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967054 2024-11-26T09:56:07,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967054) roll requested 2024-11-26T09:56:07,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967061 2024-11-26T09:56:07,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967061 2024-11-26T09:56:07,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967061) roll requested 2024-11-26T09:56:07,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967068 2024-11-26T09:56:07,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967068 2024-11-26T09:56:07,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967068) roll requested 2024-11-26T09:56:07,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967075 2024-11-26T09:56:07,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967075 2024-11-26T09:56:07,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967075) roll requested 2024-11-26T09:56:07,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967082 2024-11-26T09:56:07,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967082 2024-11-26T09:56:07,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967082) roll requested 2024-11-26T09:56:07,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967089 2024-11-26T09:56:07,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967089 2024-11-26T09:56:07,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967089) roll requested 2024-11-26T09:56:07,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967096 2024-11-26T09:56:07,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967096 2024-11-26T09:56:07,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967096) roll requested 2024-11-26T09:56:07,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967103 2024-11-26T09:56:07,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967103 2024-11-26T09:56:07,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967103) roll requested 2024-11-26T09:56:07,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967110 2024-11-26T09:56:07,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967110 2024-11-26T09:56:07,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967110) roll requested 2024-11-26T09:56:07,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967116 2024-11-26T09:56:07,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967116 2024-11-26T09:56:07,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967116) roll requested 2024-11-26T09:56:07,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967123 2024-11-26T09:56:07,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967123 2024-11-26T09:56:07,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967123) roll requested 2024-11-26T09:56:07,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967131 2024-11-26T09:56:07,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967131 2024-11-26T09:56:07,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967131) roll requested 2024-11-26T09:56:07,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967140 2024-11-26T09:56:07,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967140 2024-11-26T09:56:07,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967140) roll requested 2024-11-26T09:56:07,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967160 2024-11-26T09:56:07,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967160 2024-11-26T09:56:07,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967160) roll requested 2024-11-26T09:56:07,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967167 2024-11-26T09:56:07,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967167 2024-11-26T09:56:07,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967167) roll requested 2024-11-26T09:56:07,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967185 2024-11-26T09:56:07,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967185 2024-11-26T09:56:07,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967185) roll requested 2024-11-26T09:56:07,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967192 2024-11-26T09:56:07,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967192 2024-11-26T09:56:07,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967192) roll requested 2024-11-26T09:56:07,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967203 2024-11-26T09:56:07,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967203 2024-11-26T09:56:07,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967203) roll requested 2024-11-26T09:56:07,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967211 2024-11-26T09:56:07,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967211 2024-11-26T09:56:07,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967211) roll requested 2024-11-26T09:56:07,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967225 2024-11-26T09:56:07,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967225 2024-11-26T09:56:07,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967225) roll requested 2024-11-26T09:56:07,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967252 2024-11-26T09:56:07,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967252 2024-11-26T09:56:07,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967252) roll requested 2024-11-26T09:56:07,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967276 2024-11-26T09:56:07,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967276 2024-11-26T09:56:07,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967276) roll requested 2024-11-26T09:56:07,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967288 2024-11-26T09:56:07,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967288 2024-11-26T09:56:07,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967288) roll requested 2024-11-26T09:56:07,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967306 2024-11-26T09:56:07,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967306 2024-11-26T09:56:07,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967306) roll requested 2024-11-26T09:56:07,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967320 2024-11-26T09:56:07,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967320 2024-11-26T09:56:07,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967320) roll requested 2024-11-26T09:56:07,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967327 2024-11-26T09:56:07,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967327 2024-11-26T09:56:07,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967327) roll requested 2024-11-26T09:56:07,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967334 2024-11-26T09:56:07,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967334 2024-11-26T09:56:07,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967334) roll requested 2024-11-26T09:56:07,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967340 2024-11-26T09:56:07,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967340 2024-11-26T09:56:07,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967340) roll requested 2024-11-26T09:56:07,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967347 2024-11-26T09:56:07,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967347 2024-11-26T09:56:07,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967347) roll requested 2024-11-26T09:56:07,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967353 2024-11-26T09:56:07,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967353 2024-11-26T09:56:07,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967353) roll requested 2024-11-26T09:56:07,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967360 2024-11-26T09:56:07,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967360 2024-11-26T09:56:07,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967360) roll requested 2024-11-26T09:56:07,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967366 2024-11-26T09:56:07,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967366 2024-11-26T09:56:07,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967366) roll requested 2024-11-26T09:56:07,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967373 2024-11-26T09:56:07,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967373 2024-11-26T09:56:07,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967373) roll requested 2024-11-26T09:56:07,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967379 2024-11-26T09:56:07,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967379 2024-11-26T09:56:07,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967379) roll requested 2024-11-26T09:56:07,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967385 2024-11-26T09:56:07,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967385 2024-11-26T09:56:07,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967385) roll requested 2024-11-26T09:56:07,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967395 2024-11-26T09:56:07,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967395 2024-11-26T09:56:07,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967395) roll requested 2024-11-26T09:56:07,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967404 2024-11-26T09:56:07,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967404 2024-11-26T09:56:07,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967404) roll requested 2024-11-26T09:56:07,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967410 2024-11-26T09:56:07,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967410 2024-11-26T09:56:07,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967410) roll requested 2024-11-26T09:56:07,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967417 2024-11-26T09:56:07,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967417 2024-11-26T09:56:07,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967417) roll requested 2024-11-26T09:56:07,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967423 2024-11-26T09:56:07,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967423 2024-11-26T09:56:07,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967423) roll requested 2024-11-26T09:56:07,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967429 2024-11-26T09:56:07,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967429 2024-11-26T09:56:07,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967429) roll requested 2024-11-26T09:56:07,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967435 2024-11-26T09:56:07,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967435 2024-11-26T09:56:07,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967435) roll requested 2024-11-26T09:56:07,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967442 2024-11-26T09:56:07,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967442 2024-11-26T09:56:07,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967442) roll requested 2024-11-26T09:56:07,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967448 2024-11-26T09:56:07,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967448 2024-11-26T09:56:07,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967448) roll requested 2024-11-26T09:56:07,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967454 2024-11-26T09:56:07,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967454 2024-11-26T09:56:07,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967454) roll requested 2024-11-26T09:56:07,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967461 2024-11-26T09:56:07,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967461 2024-11-26T09:56:07,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967461) roll requested 2024-11-26T09:56:07,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967467 2024-11-26T09:56:07,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967467 2024-11-26T09:56:07,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967467) roll requested 2024-11-26T09:56:07,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967474 2024-11-26T09:56:07,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967474 2024-11-26T09:56:07,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967474) roll requested 2024-11-26T09:56:07,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967481 2024-11-26T09:56:07,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967481 2024-11-26T09:56:07,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967481) roll requested 2024-11-26T09:56:07,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967487 2024-11-26T09:56:07,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967487 2024-11-26T09:56:07,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967487) roll requested 2024-11-26T09:56:07,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967493 2024-11-26T09:56:07,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967493 2024-11-26T09:56:07,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967493) roll requested 2024-11-26T09:56:07,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967500 2024-11-26T09:56:07,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967500 2024-11-26T09:56:07,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967500) roll requested 2024-11-26T09:56:07,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967506 2024-11-26T09:56:07,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967506 2024-11-26T09:56:07,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967506) roll requested 2024-11-26T09:56:07,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967513 2024-11-26T09:56:07,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967513 2024-11-26T09:56:07,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967513) roll requested 2024-11-26T09:56:07,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967519 2024-11-26T09:56:07,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967519 2024-11-26T09:56:07,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967519) roll requested 2024-11-26T09:56:07,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967525 2024-11-26T09:56:07,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967525 2024-11-26T09:56:07,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967525) roll requested 2024-11-26T09:56:07,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967532 2024-11-26T09:56:07,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967532 2024-11-26T09:56:07,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967532) roll requested 2024-11-26T09:56:07,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967538 2024-11-26T09:56:07,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967538 2024-11-26T09:56:07,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967538) roll requested 2024-11-26T09:56:07,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967545 2024-11-26T09:56:07,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967545 2024-11-26T09:56:07,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967545) roll requested 2024-11-26T09:56:07,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967551 2024-11-26T09:56:07,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967551 2024-11-26T09:56:07,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967551) roll requested 2024-11-26T09:56:07,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967558 2024-11-26T09:56:07,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967558 2024-11-26T09:56:07,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967558) roll requested 2024-11-26T09:56:07,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967564 2024-11-26T09:56:07,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967564 2024-11-26T09:56:07,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967564) roll requested 2024-11-26T09:56:07,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967570 2024-11-26T09:56:07,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967570 2024-11-26T09:56:07,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967570) roll requested 2024-11-26T09:56:07,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967577 2024-11-26T09:56:07,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967577 2024-11-26T09:56:07,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967577) roll requested 2024-11-26T09:56:07,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967583 2024-11-26T09:56:07,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967583 2024-11-26T09:56:07,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967583) roll requested 2024-11-26T09:56:07,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967589 2024-11-26T09:56:07,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967589 2024-11-26T09:56:07,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967589) roll requested 2024-11-26T09:56:07,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967595 2024-11-26T09:56:07,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967595 2024-11-26T09:56:07,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967595) roll requested 2024-11-26T09:56:07,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967602 2024-11-26T09:56:07,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967602 2024-11-26T09:56:07,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967602) roll requested 2024-11-26T09:56:07,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967608 2024-11-26T09:56:07,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967608 2024-11-26T09:56:07,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967608) roll requested 2024-11-26T09:56:07,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967615 2024-11-26T09:56:07,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967615 2024-11-26T09:56:07,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967615) roll requested 2024-11-26T09:56:07,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967621 2024-11-26T09:56:07,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967621 2024-11-26T09:56:07,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967621) roll requested 2024-11-26T09:56:07,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967627 2024-11-26T09:56:07,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967627 2024-11-26T09:56:07,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967627) roll requested 2024-11-26T09:56:07,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967634 2024-11-26T09:56:07,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967634 2024-11-26T09:56:07,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967634) roll requested 2024-11-26T09:56:07,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967640 2024-11-26T09:56:07,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967640 2024-11-26T09:56:07,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967640) roll requested 2024-11-26T09:56:07,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967647 2024-11-26T09:56:07,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967647 2024-11-26T09:56:07,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967647) roll requested 2024-11-26T09:56:07,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967653 2024-11-26T09:56:07,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967653 2024-11-26T09:56:07,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967653) roll requested 2024-11-26T09:56:07,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967659 2024-11-26T09:56:07,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967659 2024-11-26T09:56:07,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967659) roll requested 2024-11-26T09:56:07,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967666 2024-11-26T09:56:07,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967666 2024-11-26T09:56:07,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967666) roll requested 2024-11-26T09:56:07,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967672 2024-11-26T09:56:07,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967672 2024-11-26T09:56:07,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967672) roll requested 2024-11-26T09:56:07,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967679 2024-11-26T09:56:07,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967679 2024-11-26T09:56:07,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967679) roll requested 2024-11-26T09:56:07,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967685 2024-11-26T09:56:07,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967685 2024-11-26T09:56:07,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967685) roll requested 2024-11-26T09:56:07,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967692 2024-11-26T09:56:07,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967692 2024-11-26T09:56:07,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967692) roll requested 2024-11-26T09:56:07,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967698 2024-11-26T09:56:07,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967698 2024-11-26T09:56:07,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967698) roll requested 2024-11-26T09:56:07,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967705 2024-11-26T09:56:07,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967705 2024-11-26T09:56:07,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967705) roll requested 2024-11-26T09:56:07,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967711 2024-11-26T09:56:07,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967711 2024-11-26T09:56:07,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,717 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967711) roll requested 2024-11-26T09:56:07,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967717 2024-11-26T09:56:07,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967717 2024-11-26T09:56:07,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967717) roll requested 2024-11-26T09:56:07,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967725 2024-11-26T09:56:07,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967725 2024-11-26T09:56:07,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967725) roll requested 2024-11-26T09:56:07,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967736 2024-11-26T09:56:07,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967736 2024-11-26T09:56:07,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967736) roll requested 2024-11-26T09:56:07,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967744 2024-11-26T09:56:07,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967744 2024-11-26T09:56:07,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967744) roll requested 2024-11-26T09:56:07,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967751 2024-11-26T09:56:07,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967751 2024-11-26T09:56:07,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967751) roll requested 2024-11-26T09:56:07,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967759 2024-11-26T09:56:07,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967759 2024-11-26T09:56:07,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967759) roll requested 2024-11-26T09:56:07,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967766 2024-11-26T09:56:07,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967766 2024-11-26T09:56:07,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967766) roll requested 2024-11-26T09:56:07,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967773 2024-11-26T09:56:07,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967773 2024-11-26T09:56:07,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967773) roll requested 2024-11-26T09:56:07,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967788 2024-11-26T09:56:07,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967788 2024-11-26T09:56:07,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967788) roll requested 2024-11-26T09:56:07,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967798 2024-11-26T09:56:07,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967798 2024-11-26T09:56:07,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967798) roll requested 2024-11-26T09:56:07,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967808 2024-11-26T09:56:07,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967808 2024-11-26T09:56:07,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967808) roll requested 2024-11-26T09:56:07,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967816 2024-11-26T09:56:07,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967816 2024-11-26T09:56:07,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967816) roll requested 2024-11-26T09:56:07,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967824 2024-11-26T09:56:07,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967824 2024-11-26T09:56:07,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967824) roll requested 2024-11-26T09:56:07,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967833 2024-11-26T09:56:07,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967833 2024-11-26T09:56:07,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967833) roll requested 2024-11-26T09:56:07,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967840 2024-11-26T09:56:07,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967840 2024-11-26T09:56:07,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967840) roll requested 2024-11-26T09:56:07,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967848 2024-11-26T09:56:07,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967848 2024-11-26T09:56:07,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967848) roll requested 2024-11-26T09:56:07,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967856 2024-11-26T09:56:07,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967856 2024-11-26T09:56:07,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967856) roll requested 2024-11-26T09:56:07,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967864 2024-11-26T09:56:07,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967864 2024-11-26T09:56:07,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967864) roll requested 2024-11-26T09:56:07,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967876 2024-11-26T09:56:07,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967876 2024-11-26T09:56:07,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967876) roll requested 2024-11-26T09:56:07,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967884 2024-11-26T09:56:07,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967884 2024-11-26T09:56:07,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967884) roll requested 2024-11-26T09:56:07,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967892 2024-11-26T09:56:07,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967892 2024-11-26T09:56:07,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967892) roll requested 2024-11-26T09:56:07,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967899 2024-11-26T09:56:07,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967899 2024-11-26T09:56:07,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967899) roll requested 2024-11-26T09:56:07,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967907 2024-11-26T09:56:07,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967907 2024-11-26T09:56:07,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967907) roll requested 2024-11-26T09:56:07,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967920 2024-11-26T09:56:07,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967920 2024-11-26T09:56:07,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967920) roll requested 2024-11-26T09:56:07,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967928 2024-11-26T09:56:07,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967928 2024-11-26T09:56:07,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967928) roll requested 2024-11-26T09:56:07,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967936 2024-11-26T09:56:07,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967936 2024-11-26T09:56:07,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967936) roll requested 2024-11-26T09:56:07,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967943 2024-11-26T09:56:07,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967943 2024-11-26T09:56:07,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967943) roll requested 2024-11-26T09:56:07,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967951 2024-11-26T09:56:07,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967951 2024-11-26T09:56:07,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967951) roll requested 2024-11-26T09:56:07,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967958 2024-11-26T09:56:07,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967958 2024-11-26T09:56:07,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967958) roll requested 2024-11-26T09:56:07,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967965 2024-11-26T09:56:07,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967965 2024-11-26T09:56:07,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967965) roll requested 2024-11-26T09:56:07,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967973 2024-11-26T09:56:07,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967973 2024-11-26T09:56:07,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967973) roll requested 2024-11-26T09:56:07,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967981 2024-11-26T09:56:07,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967981 2024-11-26T09:56:07,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967981) roll requested 2024-11-26T09:56:07,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967989 2024-11-26T09:56:07,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:07,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967989 2024-11-26T09:56:07,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:07,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:07,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967989) roll requested 2024-11-26T09:56:07,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614967997 2024-11-26T09:56:08,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967997 2024-11-26T09:56:08,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614967997) roll requested 2024-11-26T09:56:08,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968004 2024-11-26T09:56:08,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614967997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968004 2024-11-26T09:56:08,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968004) roll requested 2024-11-26T09:56:08,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968011 2024-11-26T09:56:08,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968011 2024-11-26T09:56:08,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968011) roll requested 2024-11-26T09:56:08,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968018 2024-11-26T09:56:08,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968018 2024-11-26T09:56:08,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968018) roll requested 2024-11-26T09:56:08,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968033 2024-11-26T09:56:08,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968033 2024-11-26T09:56:08,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968033) roll requested 2024-11-26T09:56:08,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968047 2024-11-26T09:56:08,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968047 2024-11-26T09:56:08,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968047) roll requested 2024-11-26T09:56:08,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968060 2024-11-26T09:56:08,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968060 2024-11-26T09:56:08,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968060) roll requested 2024-11-26T09:56:08,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968067 2024-11-26T09:56:08,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968067 2024-11-26T09:56:08,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968067) roll requested 2024-11-26T09:56:08,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968075 2024-11-26T09:56:08,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968075 2024-11-26T09:56:08,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968075) roll requested 2024-11-26T09:56:08,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968082 2024-11-26T09:56:08,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968082 2024-11-26T09:56:08,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968082) roll requested 2024-11-26T09:56:08,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968092 2024-11-26T09:56:08,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968092 2024-11-26T09:56:08,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968092) roll requested 2024-11-26T09:56:08,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968099 2024-11-26T09:56:08,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968099 2024-11-26T09:56:08,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968099) roll requested 2024-11-26T09:56:08,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968106 2024-11-26T09:56:08,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968106 2024-11-26T09:56:08,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968106) roll requested 2024-11-26T09:56:08,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968116 2024-11-26T09:56:08,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968116 2024-11-26T09:56:08,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968116) roll requested 2024-11-26T09:56:08,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968123 2024-11-26T09:56:08,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968123 2024-11-26T09:56:08,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968123) roll requested 2024-11-26T09:56:08,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968130 2024-11-26T09:56:08,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968130 2024-11-26T09:56:08,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968130) roll requested 2024-11-26T09:56:08,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968137 2024-11-26T09:56:08,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968137 2024-11-26T09:56:08,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968137) roll requested 2024-11-26T09:56:08,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968144 2024-11-26T09:56:08,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968144 2024-11-26T09:56:08,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968144) roll requested 2024-11-26T09:56:08,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968151 2024-11-26T09:56:08,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968151 2024-11-26T09:56:08,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968151) roll requested 2024-11-26T09:56:08,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968159 2024-11-26T09:56:08,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968159 2024-11-26T09:56:08,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968159) roll requested 2024-11-26T09:56:08,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968166 2024-11-26T09:56:08,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968166 2024-11-26T09:56:08,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968166) roll requested 2024-11-26T09:56:08,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968173 2024-11-26T09:56:08,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968173 2024-11-26T09:56:08,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968173) roll requested 2024-11-26T09:56:08,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968180 2024-11-26T09:56:08,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968180 2024-11-26T09:56:08,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968180) roll requested 2024-11-26T09:56:08,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968188 2024-11-26T09:56:08,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968188 2024-11-26T09:56:08,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968188) roll requested 2024-11-26T09:56:08,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968195 2024-11-26T09:56:08,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968195 2024-11-26T09:56:08,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968195) roll requested 2024-11-26T09:56:08,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968202 2024-11-26T09:56:08,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968202 2024-11-26T09:56:08,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968202) roll requested 2024-11-26T09:56:08,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968210 2024-11-26T09:56:08,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968210 2024-11-26T09:56:08,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968210) roll requested 2024-11-26T09:56:08,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968217 2024-11-26T09:56:08,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968217 2024-11-26T09:56:08,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968217) roll requested 2024-11-26T09:56:08,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968225 2024-11-26T09:56:08,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968225 2024-11-26T09:56:08,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968225) roll requested 2024-11-26T09:56:08,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968240 2024-11-26T09:56:08,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968240 2024-11-26T09:56:08,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968240) roll requested 2024-11-26T09:56:08,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968247 2024-11-26T09:56:08,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968247 2024-11-26T09:56:08,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968247) roll requested 2024-11-26T09:56:08,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968254 2024-11-26T09:56:08,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968254 2024-11-26T09:56:08,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968254) roll requested 2024-11-26T09:56:08,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968261 2024-11-26T09:56:08,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968261 2024-11-26T09:56:08,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968261) roll requested 2024-11-26T09:56:08,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968270 2024-11-26T09:56:08,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968270 2024-11-26T09:56:08,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968270) roll requested 2024-11-26T09:56:08,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968277 2024-11-26T09:56:08,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968277 2024-11-26T09:56:08,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968277) roll requested 2024-11-26T09:56:08,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968286 2024-11-26T09:56:08,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968286 2024-11-26T09:56:08,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968286) roll requested 2024-11-26T09:56:08,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968294 2024-11-26T09:56:08,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968294 2024-11-26T09:56:08,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968294) roll requested 2024-11-26T09:56:08,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968314 2024-11-26T09:56:08,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968314 2024-11-26T09:56:08,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968314) roll requested 2024-11-26T09:56:08,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968322 2024-11-26T09:56:08,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968322 2024-11-26T09:56:08,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968322) roll requested 2024-11-26T09:56:08,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968330 2024-11-26T09:56:08,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968330 2024-11-26T09:56:08,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968330) roll requested 2024-11-26T09:56:08,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968337 2024-11-26T09:56:08,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968337 2024-11-26T09:56:08,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968337) roll requested 2024-11-26T09:56:08,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968344 2024-11-26T09:56:08,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968344 2024-11-26T09:56:08,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968344) roll requested 2024-11-26T09:56:08,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968352 2024-11-26T09:56:08,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968352 2024-11-26T09:56:08,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968352) roll requested 2024-11-26T09:56:08,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968359 2024-11-26T09:56:08,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968359 2024-11-26T09:56:08,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968359) roll requested 2024-11-26T09:56:08,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968366 2024-11-26T09:56:08,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968366 2024-11-26T09:56:08,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968366) roll requested 2024-11-26T09:56:08,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968373 2024-11-26T09:56:08,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968373 2024-11-26T09:56:08,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968373) roll requested 2024-11-26T09:56:08,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968380 2024-11-26T09:56:08,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968380 2024-11-26T09:56:08,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968380) roll requested 2024-11-26T09:56:08,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968387 2024-11-26T09:56:08,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968387 2024-11-26T09:56:08,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968387) roll requested 2024-11-26T09:56:08,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968395 2024-11-26T09:56:08,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968395 2024-11-26T09:56:08,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968395) roll requested 2024-11-26T09:56:08,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968402 2024-11-26T09:56:08,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968402 2024-11-26T09:56:08,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968402) roll requested 2024-11-26T09:56:08,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968409 2024-11-26T09:56:08,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968409 2024-11-26T09:56:08,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968409) roll requested 2024-11-26T09:56:08,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968416 2024-11-26T09:56:08,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968416 2024-11-26T09:56:08,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968416) roll requested 2024-11-26T09:56:08,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968423 2024-11-26T09:56:08,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968423 2024-11-26T09:56:08,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968423) roll requested 2024-11-26T09:56:08,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968431 2024-11-26T09:56:08,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968431 2024-11-26T09:56:08,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968431) roll requested 2024-11-26T09:56:08,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968439 2024-11-26T09:56:08,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968439 2024-11-26T09:56:08,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968439) roll requested 2024-11-26T09:56:08,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968446 2024-11-26T09:56:08,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968446 2024-11-26T09:56:08,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968446) roll requested 2024-11-26T09:56:08,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968453 2024-11-26T09:56:08,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968453 2024-11-26T09:56:08,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968453) roll requested 2024-11-26T09:56:08,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968460 2024-11-26T09:56:08,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968460 2024-11-26T09:56:08,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968460) roll requested 2024-11-26T09:56:08,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968467 2024-11-26T09:56:08,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968467 2024-11-26T09:56:08,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968467) roll requested 2024-11-26T09:56:08,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968479 2024-11-26T09:56:08,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968479 2024-11-26T09:56:08,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968479) roll requested 2024-11-26T09:56:08,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968486 2024-11-26T09:56:08,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968486 2024-11-26T09:56:08,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968486) roll requested 2024-11-26T09:56:08,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968493 2024-11-26T09:56:08,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968493 2024-11-26T09:56:08,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968493) roll requested 2024-11-26T09:56:08,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968500 2024-11-26T09:56:08,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968500 2024-11-26T09:56:08,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968500) roll requested 2024-11-26T09:56:08,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968507 2024-11-26T09:56:08,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968507 2024-11-26T09:56:08,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968507) roll requested 2024-11-26T09:56:08,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968514 2024-11-26T09:56:08,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968514 2024-11-26T09:56:08,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968514) roll requested 2024-11-26T09:56:08,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968523 2024-11-26T09:56:08,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968523 2024-11-26T09:56:08,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968523) roll requested 2024-11-26T09:56:08,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968534 2024-11-26T09:56:08,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968534 2024-11-26T09:56:08,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968534) roll requested 2024-11-26T09:56:08,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968542 2024-11-26T09:56:08,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968542 2024-11-26T09:56:08,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968542) roll requested 2024-11-26T09:56:08,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968549 2024-11-26T09:56:08,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968549 2024-11-26T09:56:08,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968549) roll requested 2024-11-26T09:56:08,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968556 2024-11-26T09:56:08,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968556 2024-11-26T09:56:08,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968556) roll requested 2024-11-26T09:56:08,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968563 2024-11-26T09:56:08,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968563 2024-11-26T09:56:08,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968563) roll requested 2024-11-26T09:56:08,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968570 2024-11-26T09:56:08,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968570 2024-11-26T09:56:08,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968570) roll requested 2024-11-26T09:56:08,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968577 2024-11-26T09:56:08,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968577 2024-11-26T09:56:08,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968577) roll requested 2024-11-26T09:56:08,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968584 2024-11-26T09:56:08,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968584 2024-11-26T09:56:08,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968584) roll requested 2024-11-26T09:56:08,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968591 2024-11-26T09:56:08,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968591 2024-11-26T09:56:08,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968591) roll requested 2024-11-26T09:56:08,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968599 2024-11-26T09:56:08,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968599 2024-11-26T09:56:08,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968599) roll requested 2024-11-26T09:56:08,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968606 2024-11-26T09:56:08,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968606 2024-11-26T09:56:08,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968606) roll requested 2024-11-26T09:56:08,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968616 2024-11-26T09:56:08,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968616 2024-11-26T09:56:08,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968616) roll requested 2024-11-26T09:56:08,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968632 2024-11-26T09:56:08,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968632 2024-11-26T09:56:08,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968632) roll requested 2024-11-26T09:56:08,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968644 2024-11-26T09:56:08,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968644 2024-11-26T09:56:08,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968644) roll requested 2024-11-26T09:56:08,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968651 2024-11-26T09:56:08,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968651 2024-11-26T09:56:08,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968651) roll requested 2024-11-26T09:56:08,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968658 2024-11-26T09:56:08,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968658 2024-11-26T09:56:08,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968658) roll requested 2024-11-26T09:56:08,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968665 2024-11-26T09:56:08,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968665 2024-11-26T09:56:08,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968665) roll requested 2024-11-26T09:56:08,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968672 2024-11-26T09:56:08,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968672 2024-11-26T09:56:08,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968672) roll requested 2024-11-26T09:56:08,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968679 2024-11-26T09:56:08,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968679 2024-11-26T09:56:08,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968679) roll requested 2024-11-26T09:56:08,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968688 2024-11-26T09:56:08,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968688 2024-11-26T09:56:08,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968688) roll requested 2024-11-26T09:56:08,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968695 2024-11-26T09:56:08,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968695 2024-11-26T09:56:08,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968695) roll requested 2024-11-26T09:56:08,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968703 2024-11-26T09:56:08,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968703 2024-11-26T09:56:08,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968703) roll requested 2024-11-26T09:56:08,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968710 2024-11-26T09:56:08,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968710 2024-11-26T09:56:08,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968710) roll requested 2024-11-26T09:56:08,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968718 2024-11-26T09:56:08,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968718 2024-11-26T09:56:08,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968718) roll requested 2024-11-26T09:56:08,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968728 2024-11-26T09:56:08,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968728 2024-11-26T09:56:08,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968728) roll requested 2024-11-26T09:56:08,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968737 2024-11-26T09:56:08,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968737 2024-11-26T09:56:08,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968737) roll requested 2024-11-26T09:56:08,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968744 2024-11-26T09:56:08,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968744 2024-11-26T09:56:08,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968744) roll requested 2024-11-26T09:56:08,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968751 2024-11-26T09:56:08,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968751 2024-11-26T09:56:08,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968751) roll requested 2024-11-26T09:56:08,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968758 2024-11-26T09:56:08,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968758 2024-11-26T09:56:08,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968758) roll requested 2024-11-26T09:56:08,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968768 2024-11-26T09:56:08,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968768 2024-11-26T09:56:08,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968768) roll requested 2024-11-26T09:56:08,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968777 2024-11-26T09:56:08,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968777 2024-11-26T09:56:08,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968777) roll requested 2024-11-26T09:56:08,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968786 2024-11-26T09:56:08,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968786 2024-11-26T09:56:08,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968786) roll requested 2024-11-26T09:56:08,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968793 2024-11-26T09:56:08,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968793 2024-11-26T09:56:08,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968793) roll requested 2024-11-26T09:56:08,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968800 2024-11-26T09:56:08,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968800 2024-11-26T09:56:08,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968800) roll requested 2024-11-26T09:56:08,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968807 2024-11-26T09:56:08,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968807 2024-11-26T09:56:08,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968807) roll requested 2024-11-26T09:56:08,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968831 2024-11-26T09:56:08,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968831 2024-11-26T09:56:08,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968831) roll requested 2024-11-26T09:56:08,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968838 2024-11-26T09:56:08,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968838 2024-11-26T09:56:08,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968838) roll requested 2024-11-26T09:56:08,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968846 2024-11-26T09:56:08,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968846 2024-11-26T09:56:08,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968846) roll requested 2024-11-26T09:56:08,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968854 2024-11-26T09:56:08,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968854 2024-11-26T09:56:08,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968854) roll requested 2024-11-26T09:56:08,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968869 2024-11-26T09:56:08,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968869 2024-11-26T09:56:08,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968869) roll requested 2024-11-26T09:56:08,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968877 2024-11-26T09:56:08,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968877 2024-11-26T09:56:08,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968877) roll requested 2024-11-26T09:56:08,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968889 2024-11-26T09:56:08,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968889 2024-11-26T09:56:08,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968889) roll requested 2024-11-26T09:56:08,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968904 2024-11-26T09:56:08,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968904 2024-11-26T09:56:08,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968904) roll requested 2024-11-26T09:56:08,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968911 2024-11-26T09:56:08,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968911 2024-11-26T09:56:08,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968911) roll requested 2024-11-26T09:56:08,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968919 2024-11-26T09:56:08,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968919 2024-11-26T09:56:08,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968919) roll requested 2024-11-26T09:56:08,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968926 2024-11-26T09:56:08,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968926 2024-11-26T09:56:08,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968926) roll requested 2024-11-26T09:56:08,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968933 2024-11-26T09:56:08,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968933 2024-11-26T09:56:08,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968933) roll requested 2024-11-26T09:56:08,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968940 2024-11-26T09:56:08,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968940 2024-11-26T09:56:08,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968940) roll requested 2024-11-26T09:56:08,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968947 2024-11-26T09:56:08,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968947 2024-11-26T09:56:08,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968947) roll requested 2024-11-26T09:56:08,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968954 2024-11-26T09:56:08,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968954 2024-11-26T09:56:08,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968954) roll requested 2024-11-26T09:56:08,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968962 2024-11-26T09:56:08,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968962 2024-11-26T09:56:08,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968962) roll requested 2024-11-26T09:56:08,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968969 2024-11-26T09:56:08,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968969 2024-11-26T09:56:08,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968969) roll requested 2024-11-26T09:56:08,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968976 2024-11-26T09:56:08,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968976 2024-11-26T09:56:08,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968976) roll requested 2024-11-26T09:56:08,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968984 2024-11-26T09:56:08,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968984 2024-11-26T09:56:08,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968984) roll requested 2024-11-26T09:56:08,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968991 2024-11-26T09:56:08,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:08,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968991 2024-11-26T09:56:08,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:08,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:08,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968991) roll requested 2024-11-26T09:56:08,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614968998 2024-11-26T09:56:09,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968998 2024-11-26T09:56:09,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614968998) roll requested 2024-11-26T09:56:09,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969006 2024-11-26T09:56:09,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614968998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969006 2024-11-26T09:56:09,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969006) roll requested 2024-11-26T09:56:09,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969012 2024-11-26T09:56:09,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969012 2024-11-26T09:56:09,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969012) roll requested 2024-11-26T09:56:09,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969019 2024-11-26T09:56:09,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969019 2024-11-26T09:56:09,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969019) roll requested 2024-11-26T09:56:09,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969027 2024-11-26T09:56:09,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969027 2024-11-26T09:56:09,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969027) roll requested 2024-11-26T09:56:09,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969035 2024-11-26T09:56:09,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969035 2024-11-26T09:56:09,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969035) roll requested 2024-11-26T09:56:09,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969042 2024-11-26T09:56:09,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969042 2024-11-26T09:56:09,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969042) roll requested 2024-11-26T09:56:09,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969050 2024-11-26T09:56:09,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969050 2024-11-26T09:56:09,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969050) roll requested 2024-11-26T09:56:09,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969057 2024-11-26T09:56:09,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969057 2024-11-26T09:56:09,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969057) roll requested 2024-11-26T09:56:09,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969064 2024-11-26T09:56:09,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969064 2024-11-26T09:56:09,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969064) roll requested 2024-11-26T09:56:09,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969071 2024-11-26T09:56:09,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969071 2024-11-26T09:56:09,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969071) roll requested 2024-11-26T09:56:09,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969078 2024-11-26T09:56:09,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969078 2024-11-26T09:56:09,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969078) roll requested 2024-11-26T09:56:09,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969085 2024-11-26T09:56:09,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969085 2024-11-26T09:56:09,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969085) roll requested 2024-11-26T09:56:09,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969093 2024-11-26T09:56:09,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969093 2024-11-26T09:56:09,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969093) roll requested 2024-11-26T09:56:09,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969100 2024-11-26T09:56:09,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969100 2024-11-26T09:56:09,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969100) roll requested 2024-11-26T09:56:09,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969107 2024-11-26T09:56:09,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969107 2024-11-26T09:56:09,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969107) roll requested 2024-11-26T09:56:09,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969114 2024-11-26T09:56:09,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969114 2024-11-26T09:56:09,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969114) roll requested 2024-11-26T09:56:09,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969121 2024-11-26T09:56:09,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969121 2024-11-26T09:56:09,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969121) roll requested 2024-11-26T09:56:09,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969129 2024-11-26T09:56:09,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969129 2024-11-26T09:56:09,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969129) roll requested 2024-11-26T09:56:09,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969136 2024-11-26T09:56:09,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969136 2024-11-26T09:56:09,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969136) roll requested 2024-11-26T09:56:09,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969143 2024-11-26T09:56:09,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969143 2024-11-26T09:56:09,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969143) roll requested 2024-11-26T09:56:09,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969151 2024-11-26T09:56:09,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969151 2024-11-26T09:56:09,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969151) roll requested 2024-11-26T09:56:09,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969158 2024-11-26T09:56:09,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969158 2024-11-26T09:56:09,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969158) roll requested 2024-11-26T09:56:09,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969166 2024-11-26T09:56:09,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969166 2024-11-26T09:56:09,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969166) roll requested 2024-11-26T09:56:09,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969174 2024-11-26T09:56:09,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969174 2024-11-26T09:56:09,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969174) roll requested 2024-11-26T09:56:09,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969182 2024-11-26T09:56:09,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969182 2024-11-26T09:56:09,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969182) roll requested 2024-11-26T09:56:09,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969191 2024-11-26T09:56:09,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969191 2024-11-26T09:56:09,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969191) roll requested 2024-11-26T09:56:09,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969202 2024-11-26T09:56:09,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969202 2024-11-26T09:56:09,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969202) roll requested 2024-11-26T09:56:09,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969209 2024-11-26T09:56:09,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969209 2024-11-26T09:56:09,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969209) roll requested 2024-11-26T09:56:09,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969216 2024-11-26T09:56:09,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969216 2024-11-26T09:56:09,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969216) roll requested 2024-11-26T09:56:09,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969223 2024-11-26T09:56:09,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969223 2024-11-26T09:56:09,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969223) roll requested 2024-11-26T09:56:09,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969231 2024-11-26T09:56:09,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969231 2024-11-26T09:56:09,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969231) roll requested 2024-11-26T09:56:09,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969238 2024-11-26T09:56:09,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969238 2024-11-26T09:56:09,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969238) roll requested 2024-11-26T09:56:09,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969245 2024-11-26T09:56:09,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969245 2024-11-26T09:56:09,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969245) roll requested 2024-11-26T09:56:09,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969253 2024-11-26T09:56:09,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969253 2024-11-26T09:56:09,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969253) roll requested 2024-11-26T09:56:09,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969268 2024-11-26T09:56:09,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969268 2024-11-26T09:56:09,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969268) roll requested 2024-11-26T09:56:09,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969278 2024-11-26T09:56:09,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969278 2024-11-26T09:56:09,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969278) roll requested 2024-11-26T09:56:09,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969288 2024-11-26T09:56:09,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969288 2024-11-26T09:56:09,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969288) roll requested 2024-11-26T09:56:09,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969296 2024-11-26T09:56:09,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969296 2024-11-26T09:56:09,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969296) roll requested 2024-11-26T09:56:09,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969304 2024-11-26T09:56:09,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969304 2024-11-26T09:56:09,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969304) roll requested 2024-11-26T09:56:09,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969311 2024-11-26T09:56:09,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969311 2024-11-26T09:56:09,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969311) roll requested 2024-11-26T09:56:09,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969318 2024-11-26T09:56:09,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969318 2024-11-26T09:56:09,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969318) roll requested 2024-11-26T09:56:09,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969326 2024-11-26T09:56:09,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969326 2024-11-26T09:56:09,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969326) roll requested 2024-11-26T09:56:09,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969332 2024-11-26T09:56:09,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969332 2024-11-26T09:56:09,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969332) roll requested 2024-11-26T09:56:09,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969339 2024-11-26T09:56:09,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969339 2024-11-26T09:56:09,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969339) roll requested 2024-11-26T09:56:09,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969346 2024-11-26T09:56:09,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969346 2024-11-26T09:56:09,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969346) roll requested 2024-11-26T09:56:09,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969354 2024-11-26T09:56:09,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969354 2024-11-26T09:56:09,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969354) roll requested 2024-11-26T09:56:09,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969361 2024-11-26T09:56:09,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969361 2024-11-26T09:56:09,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969361) roll requested 2024-11-26T09:56:09,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969368 2024-11-26T09:56:09,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969368 2024-11-26T09:56:09,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969368) roll requested 2024-11-26T09:56:09,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969375 2024-11-26T09:56:09,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969375 2024-11-26T09:56:09,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969375) roll requested 2024-11-26T09:56:09,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969382 2024-11-26T09:56:09,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969382 2024-11-26T09:56:09,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969382) roll requested 2024-11-26T09:56:09,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969389 2024-11-26T09:56:09,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969389 2024-11-26T09:56:09,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969389) roll requested 2024-11-26T09:56:09,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969397 2024-11-26T09:56:09,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969397 2024-11-26T09:56:09,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969397) roll requested 2024-11-26T09:56:09,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969405 2024-11-26T09:56:09,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969405 2024-11-26T09:56:09,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969405) roll requested 2024-11-26T09:56:09,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969412 2024-11-26T09:56:09,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969412 2024-11-26T09:56:09,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969412) roll requested 2024-11-26T09:56:09,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969419 2024-11-26T09:56:09,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969419 2024-11-26T09:56:09,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969419) roll requested 2024-11-26T09:56:09,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969427 2024-11-26T09:56:09,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969427 2024-11-26T09:56:09,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969427) roll requested 2024-11-26T09:56:09,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969435 2024-11-26T09:56:09,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969435 2024-11-26T09:56:09,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969435) roll requested 2024-11-26T09:56:09,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969442 2024-11-26T09:56:09,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969442 2024-11-26T09:56:09,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969442) roll requested 2024-11-26T09:56:09,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969450 2024-11-26T09:56:09,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969450 2024-11-26T09:56:09,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969450) roll requested 2024-11-26T09:56:09,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969456 2024-11-26T09:56:09,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969456 2024-11-26T09:56:09,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969456) roll requested 2024-11-26T09:56:09,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969463 2024-11-26T09:56:09,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969463 2024-11-26T09:56:09,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969463) roll requested 2024-11-26T09:56:09,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969470 2024-11-26T09:56:09,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969470 2024-11-26T09:56:09,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969470) roll requested 2024-11-26T09:56:09,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969477 2024-11-26T09:56:09,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969477 2024-11-26T09:56:09,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969477) roll requested 2024-11-26T09:56:09,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969485 2024-11-26T09:56:09,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969485 2024-11-26T09:56:09,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969485) roll requested 2024-11-26T09:56:09,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969492 2024-11-26T09:56:09,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969492 2024-11-26T09:56:09,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969492) roll requested 2024-11-26T09:56:09,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969500 2024-11-26T09:56:09,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969500 2024-11-26T09:56:09,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969500) roll requested 2024-11-26T09:56:09,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969507 2024-11-26T09:56:09,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969507 2024-11-26T09:56:09,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969507) roll requested 2024-11-26T09:56:09,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969515 2024-11-26T09:56:09,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969515 2024-11-26T09:56:09,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969515) roll requested 2024-11-26T09:56:09,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969525 2024-11-26T09:56:09,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969525 2024-11-26T09:56:09,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969525) roll requested 2024-11-26T09:56:09,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969532 2024-11-26T09:56:09,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969532 2024-11-26T09:56:09,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969532) roll requested 2024-11-26T09:56:09,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969540 2024-11-26T09:56:09,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969540 2024-11-26T09:56:09,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969540) roll requested 2024-11-26T09:56:09,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969547 2024-11-26T09:56:09,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969547 2024-11-26T09:56:09,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969547) roll requested 2024-11-26T09:56:09,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969554 2024-11-26T09:56:09,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969554 2024-11-26T09:56:09,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969554) roll requested 2024-11-26T09:56:09,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969561 2024-11-26T09:56:09,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969561 2024-11-26T09:56:09,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969561) roll requested 2024-11-26T09:56:09,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969568 2024-11-26T09:56:09,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969568 2024-11-26T09:56:09,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969568) roll requested 2024-11-26T09:56:09,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969575 2024-11-26T09:56:09,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969575 2024-11-26T09:56:09,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969575) roll requested 2024-11-26T09:56:09,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969584 2024-11-26T09:56:09,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969584 2024-11-26T09:56:09,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969584) roll requested 2024-11-26T09:56:09,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969591 2024-11-26T09:56:09,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969591 2024-11-26T09:56:09,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969591) roll requested 2024-11-26T09:56:09,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969599 2024-11-26T09:56:09,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969599 2024-11-26T09:56:09,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969599) roll requested 2024-11-26T09:56:09,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969606 2024-11-26T09:56:09,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969606 2024-11-26T09:56:09,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969606) roll requested 2024-11-26T09:56:09,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969625 2024-11-26T09:56:09,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969625 2024-11-26T09:56:09,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969625) roll requested 2024-11-26T09:56:09,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969644 2024-11-26T09:56:09,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969644 2024-11-26T09:56:09,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969644) roll requested 2024-11-26T09:56:09,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969651 2024-11-26T09:56:09,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969651 2024-11-26T09:56:09,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969651) roll requested 2024-11-26T09:56:09,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969659 2024-11-26T09:56:09,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969659 2024-11-26T09:56:09,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969659) roll requested 2024-11-26T09:56:09,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969666 2024-11-26T09:56:09,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969666 2024-11-26T09:56:09,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969666) roll requested 2024-11-26T09:56:09,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969672 2024-11-26T09:56:09,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969672 2024-11-26T09:56:09,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969672) roll requested 2024-11-26T09:56:09,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969679 2024-11-26T09:56:09,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969679 2024-11-26T09:56:09,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969679) roll requested 2024-11-26T09:56:09,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969686 2024-11-26T09:56:09,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969686 2024-11-26T09:56:09,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969686) roll requested 2024-11-26T09:56:09,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969694 2024-11-26T09:56:09,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969694 2024-11-26T09:56:09,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969694) roll requested 2024-11-26T09:56:09,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969701 2024-11-26T09:56:09,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969701 2024-11-26T09:56:09,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969701) roll requested 2024-11-26T09:56:09,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969707 2024-11-26T09:56:09,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969707 2024-11-26T09:56:09,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969707) roll requested 2024-11-26T09:56:09,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969715 2024-11-26T09:56:09,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969715 2024-11-26T09:56:09,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969715) roll requested 2024-11-26T09:56:09,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969723 2024-11-26T09:56:09,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969723 2024-11-26T09:56:09,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,730 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969723) roll requested 2024-11-26T09:56:09,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969731 2024-11-26T09:56:09,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969731 2024-11-26T09:56:09,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969731) roll requested 2024-11-26T09:56:09,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969737 2024-11-26T09:56:09,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969731 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969737 2024-11-26T09:56:09,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969737) roll requested 2024-11-26T09:56:09,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969744 2024-11-26T09:56:09,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969744 2024-11-26T09:56:09,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969744) roll requested 2024-11-26T09:56:09,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969751 2024-11-26T09:56:09,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969751 2024-11-26T09:56:09,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969751) roll requested 2024-11-26T09:56:09,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969758 2024-11-26T09:56:09,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969758 2024-11-26T09:56:09,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969758) roll requested 2024-11-26T09:56:09,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969765 2024-11-26T09:56:09,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969765 2024-11-26T09:56:09,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969765) roll requested 2024-11-26T09:56:09,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969773 2024-11-26T09:56:09,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969773 2024-11-26T09:56:09,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969773) roll requested 2024-11-26T09:56:09,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969780 2024-11-26T09:56:09,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969780 2024-11-26T09:56:09,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969780) roll requested 2024-11-26T09:56:09,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969787 2024-11-26T09:56:09,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969787 2024-11-26T09:56:09,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969787) roll requested 2024-11-26T09:56:09,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969793 2024-11-26T09:56:09,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969793 2024-11-26T09:56:09,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969793) roll requested 2024-11-26T09:56:09,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969800 2024-11-26T09:56:09,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969800 2024-11-26T09:56:09,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969800) roll requested 2024-11-26T09:56:09,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969808 2024-11-26T09:56:09,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969808 2024-11-26T09:56:09,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969808) roll requested 2024-11-26T09:56:09,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969815 2024-11-26T09:56:09,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969815 2024-11-26T09:56:09,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969815) roll requested 2024-11-26T09:56:09,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969822 2024-11-26T09:56:09,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969822 2024-11-26T09:56:09,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969822) roll requested 2024-11-26T09:56:09,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969830 2024-11-26T09:56:09,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969830 2024-11-26T09:56:09,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969830) roll requested 2024-11-26T09:56:09,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969837 2024-11-26T09:56:09,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969837 2024-11-26T09:56:09,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969837) roll requested 2024-11-26T09:56:09,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969848 2024-11-26T09:56:09,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969848 2024-11-26T09:56:09,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969848) roll requested 2024-11-26T09:56:09,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969856 2024-11-26T09:56:09,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969856 2024-11-26T09:56:09,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969856) roll requested 2024-11-26T09:56:09,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969867 2024-11-26T09:56:09,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969867 2024-11-26T09:56:09,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969867) roll requested 2024-11-26T09:56:09,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969874 2024-11-26T09:56:09,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969874 2024-11-26T09:56:09,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969874) roll requested 2024-11-26T09:56:09,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969881 2024-11-26T09:56:09,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969881 2024-11-26T09:56:09,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969881) roll requested 2024-11-26T09:56:09,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969888 2024-11-26T09:56:09,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969888 2024-11-26T09:56:09,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969888) roll requested 2024-11-26T09:56:09,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969895 2024-11-26T09:56:09,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969895 2024-11-26T09:56:09,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969895) roll requested 2024-11-26T09:56:09,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969906 2024-11-26T09:56:09,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969906 2024-11-26T09:56:09,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969906) roll requested 2024-11-26T09:56:09,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969914 2024-11-26T09:56:09,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969914 2024-11-26T09:56:09,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969914) roll requested 2024-11-26T09:56:09,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969922 2024-11-26T09:56:09,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969922 2024-11-26T09:56:09,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969922) roll requested 2024-11-26T09:56:09,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969929 2024-11-26T09:56:09,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969929 2024-11-26T09:56:09,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969929) roll requested 2024-11-26T09:56:09,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969936 2024-11-26T09:56:09,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969936 2024-11-26T09:56:09,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969936) roll requested 2024-11-26T09:56:09,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969944 2024-11-26T09:56:09,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969944 2024-11-26T09:56:09,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969944) roll requested 2024-11-26T09:56:09,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969951 2024-11-26T09:56:09,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969951 2024-11-26T09:56:09,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969951) roll requested 2024-11-26T09:56:09,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969958 2024-11-26T09:56:09,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969958 2024-11-26T09:56:09,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969958) roll requested 2024-11-26T09:56:09,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969965 2024-11-26T09:56:09,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969965 2024-11-26T09:56:09,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969965) roll requested 2024-11-26T09:56:09,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969972 2024-11-26T09:56:09,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969972 2024-11-26T09:56:09,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969972) roll requested 2024-11-26T09:56:09,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969980 2024-11-26T09:56:09,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969980 2024-11-26T09:56:09,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969980) roll requested 2024-11-26T09:56:09,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969987 2024-11-26T09:56:09,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:09,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969987 2024-11-26T09:56:09,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:09,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:09,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969987) roll requested 2024-11-26T09:56:09,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614969994 2024-11-26T09:56:10,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969994 2024-11-26T09:56:10,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614969994) roll requested 2024-11-26T09:56:10,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970001 2024-11-26T09:56:10,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614969994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970001 2024-11-26T09:56:10,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970001) roll requested 2024-11-26T09:56:10,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970009 2024-11-26T09:56:10,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970009 2024-11-26T09:56:10,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970009) roll requested 2024-11-26T09:56:10,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970015 2024-11-26T09:56:10,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970015 2024-11-26T09:56:10,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970015) roll requested 2024-11-26T09:56:10,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970022 2024-11-26T09:56:10,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970022 2024-11-26T09:56:10,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970022) roll requested 2024-11-26T09:56:10,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970030 2024-11-26T09:56:10,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970030 2024-11-26T09:56:10,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970030) roll requested 2024-11-26T09:56:10,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970036 2024-11-26T09:56:10,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970036 2024-11-26T09:56:10,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970036) roll requested 2024-11-26T09:56:10,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970044 2024-11-26T09:56:10,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970044 2024-11-26T09:56:10,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970044) roll requested 2024-11-26T09:56:10,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970054 2024-11-26T09:56:10,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970054 2024-11-26T09:56:10,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970054) roll requested 2024-11-26T09:56:10,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970061 2024-11-26T09:56:10,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970061 2024-11-26T09:56:10,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970061) roll requested 2024-11-26T09:56:10,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970068 2024-11-26T09:56:10,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970068 2024-11-26T09:56:10,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970068) roll requested 2024-11-26T09:56:10,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970076 2024-11-26T09:56:10,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970076 2024-11-26T09:56:10,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970076) roll requested 2024-11-26T09:56:10,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970083 2024-11-26T09:56:10,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970083 2024-11-26T09:56:10,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970083) roll requested 2024-11-26T09:56:10,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970090 2024-11-26T09:56:10,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970090 2024-11-26T09:56:10,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970090) roll requested 2024-11-26T09:56:10,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970099 2024-11-26T09:56:10,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970099 2024-11-26T09:56:10,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970099) roll requested 2024-11-26T09:56:10,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970106 2024-11-26T09:56:10,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970106 2024-11-26T09:56:10,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970106) roll requested 2024-11-26T09:56:10,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970113 2024-11-26T09:56:10,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970113 2024-11-26T09:56:10,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970113) roll requested 2024-11-26T09:56:10,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970121 2024-11-26T09:56:10,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970121 2024-11-26T09:56:10,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970121) roll requested 2024-11-26T09:56:10,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970129 2024-11-26T09:56:10,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970129 2024-11-26T09:56:10,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970129) roll requested 2024-11-26T09:56:10,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970140 2024-11-26T09:56:10,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970140 2024-11-26T09:56:10,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970140) roll requested 2024-11-26T09:56:10,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970148 2024-11-26T09:56:10,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970148 2024-11-26T09:56:10,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970148) roll requested 2024-11-26T09:56:10,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970156 2024-11-26T09:56:10,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970156 2024-11-26T09:56:10,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970156) roll requested 2024-11-26T09:56:10,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970163 2024-11-26T09:56:10,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970163 2024-11-26T09:56:10,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970163) roll requested 2024-11-26T09:56:10,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970172 2024-11-26T09:56:10,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970172 2024-11-26T09:56:10,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970172) roll requested 2024-11-26T09:56:10,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970179 2024-11-26T09:56:10,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970179 2024-11-26T09:56:10,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970179) roll requested 2024-11-26T09:56:10,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970187 2024-11-26T09:56:10,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970187 2024-11-26T09:56:10,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970187) roll requested 2024-11-26T09:56:10,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970194 2024-11-26T09:56:10,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970194 2024-11-26T09:56:10,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970194) roll requested 2024-11-26T09:56:10,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970201 2024-11-26T09:56:10,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970201 2024-11-26T09:56:10,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970201) roll requested 2024-11-26T09:56:10,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970208 2024-11-26T09:56:10,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970208 2024-11-26T09:56:10,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970208) roll requested 2024-11-26T09:56:10,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970214 2024-11-26T09:56:10,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970214 2024-11-26T09:56:10,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970214) roll requested 2024-11-26T09:56:10,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970224 2024-11-26T09:56:10,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970224 2024-11-26T09:56:10,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970224) roll requested 2024-11-26T09:56:10,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970231 2024-11-26T09:56:10,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970231 2024-11-26T09:56:10,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970231) roll requested 2024-11-26T09:56:10,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970239 2024-11-26T09:56:10,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970239 2024-11-26T09:56:10,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970239) roll requested 2024-11-26T09:56:10,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970246 2024-11-26T09:56:10,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970246 2024-11-26T09:56:10,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970246) roll requested 2024-11-26T09:56:10,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970257 2024-11-26T09:56:10,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970257 2024-11-26T09:56:10,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970257) roll requested 2024-11-26T09:56:10,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970264 2024-11-26T09:56:10,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970264 2024-11-26T09:56:10,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970264) roll requested 2024-11-26T09:56:10,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970272 2024-11-26T09:56:10,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970272 2024-11-26T09:56:10,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970272) roll requested 2024-11-26T09:56:10,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970279 2024-11-26T09:56:10,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970279 2024-11-26T09:56:10,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970279) roll requested 2024-11-26T09:56:10,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970286 2024-11-26T09:56:10,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970286 2024-11-26T09:56:10,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970286) roll requested 2024-11-26T09:56:10,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970293 2024-11-26T09:56:10,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970293 2024-11-26T09:56:10,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970293) roll requested 2024-11-26T09:56:10,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970300 2024-11-26T09:56:10,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970300 2024-11-26T09:56:10,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970300) roll requested 2024-11-26T09:56:10,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970308 2024-11-26T09:56:10,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970308 2024-11-26T09:56:10,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970308) roll requested 2024-11-26T09:56:10,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970315 2024-11-26T09:56:10,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970315 2024-11-26T09:56:10,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970315) roll requested 2024-11-26T09:56:10,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970322 2024-11-26T09:56:10,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970322 2024-11-26T09:56:10,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970322) roll requested 2024-11-26T09:56:10,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970328 2024-11-26T09:56:10,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970328 2024-11-26T09:56:10,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970328) roll requested 2024-11-26T09:56:10,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970336 2024-11-26T09:56:10,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970336 2024-11-26T09:56:10,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970336) roll requested 2024-11-26T09:56:10,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970343 2024-11-26T09:56:10,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970343 2024-11-26T09:56:10,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970343) roll requested 2024-11-26T09:56:10,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970350 2024-11-26T09:56:10,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970350 2024-11-26T09:56:10,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970350) roll requested 2024-11-26T09:56:10,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970357 2024-11-26T09:56:10,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970357 2024-11-26T09:56:10,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970357) roll requested 2024-11-26T09:56:10,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970368 2024-11-26T09:56:10,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970368 2024-11-26T09:56:10,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970368) roll requested 2024-11-26T09:56:10,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970375 2024-11-26T09:56:10,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970375 2024-11-26T09:56:10,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970375) roll requested 2024-11-26T09:56:10,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970383 2024-11-26T09:56:10,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970383 2024-11-26T09:56:10,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970383) roll requested 2024-11-26T09:56:10,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970390 2024-11-26T09:56:10,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970390 2024-11-26T09:56:10,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970390) roll requested 2024-11-26T09:56:10,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970397 2024-11-26T09:56:10,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970397 2024-11-26T09:56:10,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970397) roll requested 2024-11-26T09:56:10,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970405 2024-11-26T09:56:10,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970405 2024-11-26T09:56:10,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970405) roll requested 2024-11-26T09:56:10,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970413 2024-11-26T09:56:10,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970413 2024-11-26T09:56:10,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970413) roll requested 2024-11-26T09:56:10,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970421 2024-11-26T09:56:10,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970421 2024-11-26T09:56:10,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970421) roll requested 2024-11-26T09:56:10,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970431 2024-11-26T09:56:10,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970431 2024-11-26T09:56:10,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970431) roll requested 2024-11-26T09:56:10,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970444 2024-11-26T09:56:10,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970444 2024-11-26T09:56:10,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970444) roll requested 2024-11-26T09:56:10,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970450 2024-11-26T09:56:10,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970450 2024-11-26T09:56:10,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970450) roll requested 2024-11-26T09:56:10,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970458 2024-11-26T09:56:10,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970458 2024-11-26T09:56:10,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970458) roll requested 2024-11-26T09:56:10,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970465 2024-11-26T09:56:10,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970465 2024-11-26T09:56:10,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970465) roll requested 2024-11-26T09:56:10,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970474 2024-11-26T09:56:10,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970474 2024-11-26T09:56:10,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970474) roll requested 2024-11-26T09:56:10,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970481 2024-11-26T09:56:10,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970481 2024-11-26T09:56:10,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970481) roll requested 2024-11-26T09:56:10,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970489 2024-11-26T09:56:10,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970489 2024-11-26T09:56:10,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970489) roll requested 2024-11-26T09:56:10,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970496 2024-11-26T09:56:10,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970496 2024-11-26T09:56:10,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970496) roll requested 2024-11-26T09:56:10,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970503 2024-11-26T09:56:10,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970503 2024-11-26T09:56:10,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970503) roll requested 2024-11-26T09:56:10,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970509 2024-11-26T09:56:10,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970509 2024-11-26T09:56:10,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970509) roll requested 2024-11-26T09:56:10,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970515 2024-11-26T09:56:10,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970515 2024-11-26T09:56:10,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970515) roll requested 2024-11-26T09:56:10,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970521 2024-11-26T09:56:10,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970521 2024-11-26T09:56:10,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970521) roll requested 2024-11-26T09:56:10,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970528 2024-11-26T09:56:10,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970528 2024-11-26T09:56:10,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970528) roll requested 2024-11-26T09:56:10,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970534 2024-11-26T09:56:10,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970534 2024-11-26T09:56:10,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970534) roll requested 2024-11-26T09:56:10,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970541 2024-11-26T09:56:10,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970541 2024-11-26T09:56:10,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970541) roll requested 2024-11-26T09:56:10,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970547 2024-11-26T09:56:10,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970547 2024-11-26T09:56:10,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970547) roll requested 2024-11-26T09:56:10,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970553 2024-11-26T09:56:10,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970553 2024-11-26T09:56:10,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970553) roll requested 2024-11-26T09:56:10,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970563 2024-11-26T09:56:10,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970563 2024-11-26T09:56:10,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970563) roll requested 2024-11-26T09:56:10,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970571 2024-11-26T09:56:10,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970571 2024-11-26T09:56:10,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970571) roll requested 2024-11-26T09:56:10,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970577 2024-11-26T09:56:10,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970577 2024-11-26T09:56:10,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970577) roll requested 2024-11-26T09:56:10,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970584 2024-11-26T09:56:10,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970584 2024-11-26T09:56:10,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970584) roll requested 2024-11-26T09:56:10,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970592 2024-11-26T09:56:10,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970592 2024-11-26T09:56:10,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970592) roll requested 2024-11-26T09:56:10,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970598 2024-11-26T09:56:10,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970598 2024-11-26T09:56:10,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970598) roll requested 2024-11-26T09:56:10,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970605 2024-11-26T09:56:10,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970605 2024-11-26T09:56:10,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970605) roll requested 2024-11-26T09:56:10,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970611 2024-11-26T09:56:10,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970611 2024-11-26T09:56:10,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970611) roll requested 2024-11-26T09:56:10,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970617 2024-11-26T09:56:10,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970617 2024-11-26T09:56:10,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970617) roll requested 2024-11-26T09:56:10,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970623 2024-11-26T09:56:10,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970623 2024-11-26T09:56:10,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970623) roll requested 2024-11-26T09:56:10,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970633 2024-11-26T09:56:10,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970633 2024-11-26T09:56:10,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970633) roll requested 2024-11-26T09:56:10,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970639 2024-11-26T09:56:10,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970639 2024-11-26T09:56:10,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970639) roll requested 2024-11-26T09:56:10,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970646 2024-11-26T09:56:10,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970646 2024-11-26T09:56:10,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970646) roll requested 2024-11-26T09:56:10,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970653 2024-11-26T09:56:10,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970653 2024-11-26T09:56:10,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970653) roll requested 2024-11-26T09:56:10,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970660 2024-11-26T09:56:10,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970660 2024-11-26T09:56:10,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970660) roll requested 2024-11-26T09:56:10,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970667 2024-11-26T09:56:10,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970667 2024-11-26T09:56:10,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970667) roll requested 2024-11-26T09:56:10,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970674 2024-11-26T09:56:10,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970674 2024-11-26T09:56:10,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970674) roll requested 2024-11-26T09:56:10,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970680 2024-11-26T09:56:10,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970680 2024-11-26T09:56:10,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970680) roll requested 2024-11-26T09:56:10,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970688 2024-11-26T09:56:10,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970688 2024-11-26T09:56:10,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970688) roll requested 2024-11-26T09:56:10,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970695 2024-11-26T09:56:10,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970695 2024-11-26T09:56:10,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970695) roll requested 2024-11-26T09:56:10,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970701 2024-11-26T09:56:10,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970701 2024-11-26T09:56:10,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970701) roll requested 2024-11-26T09:56:10,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970708 2024-11-26T09:56:10,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970708 2024-11-26T09:56:10,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970708) roll requested 2024-11-26T09:56:10,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970714 2024-11-26T09:56:10,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970714 2024-11-26T09:56:10,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970714) roll requested 2024-11-26T09:56:10,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970720 2024-11-26T09:56:10,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970720 2024-11-26T09:56:10,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,730 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970720) roll requested 2024-11-26T09:56:10,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970730 2024-11-26T09:56:10,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970730 2024-11-26T09:56:10,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970730) roll requested 2024-11-26T09:56:10,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970736 2024-11-26T09:56:10,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970736 2024-11-26T09:56:10,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970736) roll requested 2024-11-26T09:56:10,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970745 2024-11-26T09:56:10,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970745 2024-11-26T09:56:10,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970745) roll requested 2024-11-26T09:56:10,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970751 2024-11-26T09:56:10,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970751 2024-11-26T09:56:10,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970751) roll requested 2024-11-26T09:56:10,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970757 2024-11-26T09:56:10,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970757 2024-11-26T09:56:10,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970757) roll requested 2024-11-26T09:56:10,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970764 2024-11-26T09:56:10,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970764 2024-11-26T09:56:10,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970764) roll requested 2024-11-26T09:56:10,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970770 2024-11-26T09:56:10,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970770 2024-11-26T09:56:10,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970770) roll requested 2024-11-26T09:56:10,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970777 2024-11-26T09:56:10,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970777 2024-11-26T09:56:10,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970777) roll requested 2024-11-26T09:56:10,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970783 2024-11-26T09:56:10,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970783 2024-11-26T09:56:10,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970783) roll requested 2024-11-26T09:56:10,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970789 2024-11-26T09:56:10,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970789 2024-11-26T09:56:10,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970789) roll requested 2024-11-26T09:56:10,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970796 2024-11-26T09:56:10,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970796 2024-11-26T09:56:10,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970796) roll requested 2024-11-26T09:56:10,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970802 2024-11-26T09:56:10,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970802 2024-11-26T09:56:10,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970802) roll requested 2024-11-26T09:56:10,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970808 2024-11-26T09:56:10,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970808 2024-11-26T09:56:10,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970808) roll requested 2024-11-26T09:56:10,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970815 2024-11-26T09:56:10,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970815 2024-11-26T09:56:10,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970815) roll requested 2024-11-26T09:56:10,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970821 2024-11-26T09:56:10,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970821 2024-11-26T09:56:10,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970821) roll requested 2024-11-26T09:56:10,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970827 2024-11-26T09:56:10,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970827 2024-11-26T09:56:10,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970827) roll requested 2024-11-26T09:56:10,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970834 2024-11-26T09:56:10,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970834 2024-11-26T09:56:10,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970834) roll requested 2024-11-26T09:56:10,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970840 2024-11-26T09:56:10,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970840 2024-11-26T09:56:10,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970840) roll requested 2024-11-26T09:56:10,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970846 2024-11-26T09:56:10,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970846 2024-11-26T09:56:10,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970846) roll requested 2024-11-26T09:56:10,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970852 2024-11-26T09:56:10,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970852 2024-11-26T09:56:10,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970852) roll requested 2024-11-26T09:56:10,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970859 2024-11-26T09:56:10,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970859 2024-11-26T09:56:10,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970859) roll requested 2024-11-26T09:56:10,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970865 2024-11-26T09:56:10,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970865 2024-11-26T09:56:10,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970865) roll requested 2024-11-26T09:56:10,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970871 2024-11-26T09:56:10,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970871 2024-11-26T09:56:10,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970871) roll requested 2024-11-26T09:56:10,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970878 2024-11-26T09:56:10,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970878 2024-11-26T09:56:10,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970878) roll requested 2024-11-26T09:56:10,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970885 2024-11-26T09:56:10,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970885 2024-11-26T09:56:10,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970885) roll requested 2024-11-26T09:56:10,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970891 2024-11-26T09:56:10,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970891 2024-11-26T09:56:10,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970891) roll requested 2024-11-26T09:56:10,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970898 2024-11-26T09:56:10,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970898 2024-11-26T09:56:10,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970898) roll requested 2024-11-26T09:56:10,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970905 2024-11-26T09:56:10,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970905 2024-11-26T09:56:10,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970905) roll requested 2024-11-26T09:56:10,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970911 2024-11-26T09:56:10,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970911 2024-11-26T09:56:10,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970911) roll requested 2024-11-26T09:56:10,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970918 2024-11-26T09:56:10,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970918 2024-11-26T09:56:10,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970911 is not closed yet, will try archiving it next time 2024-11-26T09:56:10,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970918) roll requested 2024-11-26T09:56:10,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970924 2024-11-26T09:56:10,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970924 2024-11-26T09:56:10,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970924) roll requested 2024-11-26T09:56:10,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970931 2024-11-26T09:56:10,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970931 2024-11-26T09:56:10,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970931) roll requested 2024-11-26T09:56:10,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970937 2024-11-26T09:56:10,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970937 2024-11-26T09:56:10,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970937) roll requested 2024-11-26T09:56:10,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970943 2024-11-26T09:56:10,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970943 2024-11-26T09:56:10,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970943) roll requested 2024-11-26T09:56:10,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970950 2024-11-26T09:56:10,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970950 2024-11-26T09:56:10,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970950) roll requested 2024-11-26T09:56:10,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970956 2024-11-26T09:56:10,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970956 2024-11-26T09:56:10,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970956) roll requested 2024-11-26T09:56:10,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970962 2024-11-26T09:56:10,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970962 2024-11-26T09:56:10,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970962) roll requested 2024-11-26T09:56:10,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970968 2024-11-26T09:56:10,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970968 2024-11-26T09:56:10,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970968) roll requested 2024-11-26T09:56:10,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970975 2024-11-26T09:56:10,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970975 2024-11-26T09:56:10,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970975) roll requested 2024-11-26T09:56:10,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970981 2024-11-26T09:56:10,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970981 2024-11-26T09:56:10,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970981) roll requested 2024-11-26T09:56:10,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970987 2024-11-26T09:56:10,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970987 2024-11-26T09:56:10,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970987) roll requested 2024-11-26T09:56:10,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614970993 2024-11-26T09:56:10,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:10,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970993 2024-11-26T09:56:10,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:10,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:10,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614970993) roll requested 2024-11-26T09:56:11,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971000 2024-11-26T09:56:11,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614970993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971000 2024-11-26T09:56:11,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971000) roll requested 2024-11-26T09:56:11,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971006 2024-11-26T09:56:11,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971006 2024-11-26T09:56:11,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971006) roll requested 2024-11-26T09:56:11,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971013 2024-11-26T09:56:11,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971013 2024-11-26T09:56:11,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971013) roll requested 2024-11-26T09:56:11,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971019 2024-11-26T09:56:11,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971019 2024-11-26T09:56:11,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971019) roll requested 2024-11-26T09:56:11,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971025 2024-11-26T09:56:11,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971025 2024-11-26T09:56:11,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971025) roll requested 2024-11-26T09:56:11,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971031 2024-11-26T09:56:11,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971031 2024-11-26T09:56:11,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971031) roll requested 2024-11-26T09:56:11,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971037 2024-11-26T09:56:11,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971037 2024-11-26T09:56:11,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971037) roll requested 2024-11-26T09:56:11,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971044 2024-11-26T09:56:11,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971044 2024-11-26T09:56:11,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971044) roll requested 2024-11-26T09:56:11,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971050 2024-11-26T09:56:11,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971050 2024-11-26T09:56:11,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971050) roll requested 2024-11-26T09:56:11,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971056 2024-11-26T09:56:11,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971056 2024-11-26T09:56:11,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971056) roll requested 2024-11-26T09:56:11,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971063 2024-11-26T09:56:11,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971063 2024-11-26T09:56:11,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971063) roll requested 2024-11-26T09:56:11,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971069 2024-11-26T09:56:11,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971069 2024-11-26T09:56:11,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971069) roll requested 2024-11-26T09:56:11,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971076 2024-11-26T09:56:11,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971076 2024-11-26T09:56:11,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971076) roll requested 2024-11-26T09:56:11,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971082 2024-11-26T09:56:11,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971082 2024-11-26T09:56:11,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971082) roll requested 2024-11-26T09:56:11,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971088 2024-11-26T09:56:11,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971088 2024-11-26T09:56:11,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971088) roll requested 2024-11-26T09:56:11,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971094 2024-11-26T09:56:11,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971094 2024-11-26T09:56:11,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971094) roll requested 2024-11-26T09:56:11,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971101 2024-11-26T09:56:11,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971101 2024-11-26T09:56:11,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971101) roll requested 2024-11-26T09:56:11,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971107 2024-11-26T09:56:11,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971107 2024-11-26T09:56:11,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971107) roll requested 2024-11-26T09:56:11,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971113 2024-11-26T09:56:11,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971113 2024-11-26T09:56:11,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971113) roll requested 2024-11-26T09:56:11,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971120 2024-11-26T09:56:11,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971120 2024-11-26T09:56:11,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971120) roll requested 2024-11-26T09:56:11,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971126 2024-11-26T09:56:11,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971126 2024-11-26T09:56:11,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971126) roll requested 2024-11-26T09:56:11,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971132 2024-11-26T09:56:11,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971132 2024-11-26T09:56:11,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971132) roll requested 2024-11-26T09:56:11,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971140 2024-11-26T09:56:11,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971132 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971140 2024-11-26T09:56:11,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971140) roll requested 2024-11-26T09:56:11,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971146 2024-11-26T09:56:11,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971146 2024-11-26T09:56:11,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971146) roll requested 2024-11-26T09:56:11,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971153 2024-11-26T09:56:11,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971153 2024-11-26T09:56:11,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971153) roll requested 2024-11-26T09:56:11,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971159 2024-11-26T09:56:11,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971159 2024-11-26T09:56:11,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971159) roll requested 2024-11-26T09:56:11,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971165 2024-11-26T09:56:11,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971165 2024-11-26T09:56:11,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971165) roll requested 2024-11-26T09:56:11,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971171 2024-11-26T09:56:11,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971171 2024-11-26T09:56:11,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971171) roll requested 2024-11-26T09:56:11,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971177 2024-11-26T09:56:11,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971177 2024-11-26T09:56:11,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971177) roll requested 2024-11-26T09:56:11,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971183 2024-11-26T09:56:11,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971183 2024-11-26T09:56:11,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971183) roll requested 2024-11-26T09:56:11,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971190 2024-11-26T09:56:11,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971190 2024-11-26T09:56:11,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971190) roll requested 2024-11-26T09:56:11,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971196 2024-11-26T09:56:11,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971196 2024-11-26T09:56:11,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971196) roll requested 2024-11-26T09:56:11,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971207 2024-11-26T09:56:11,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971207 2024-11-26T09:56:11,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971207) roll requested 2024-11-26T09:56:11,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971217 2024-11-26T09:56:11,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971217 2024-11-26T09:56:11,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971217) roll requested 2024-11-26T09:56:11,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971232 2024-11-26T09:56:11,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971232 2024-11-26T09:56:11,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971232) roll requested 2024-11-26T09:56:11,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971249 2024-11-26T09:56:11,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971249 2024-11-26T09:56:11,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971249) roll requested 2024-11-26T09:56:11,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971256 2024-11-26T09:56:11,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971256 2024-11-26T09:56:11,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971256) roll requested 2024-11-26T09:56:11,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971264 2024-11-26T09:56:11,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971264 2024-11-26T09:56:11,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971264) roll requested 2024-11-26T09:56:11,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971272 2024-11-26T09:56:11,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971272 2024-11-26T09:56:11,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971272) roll requested 2024-11-26T09:56:11,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971280 2024-11-26T09:56:11,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971280 2024-11-26T09:56:11,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971280) roll requested 2024-11-26T09:56:11,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971287 2024-11-26T09:56:11,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971287 2024-11-26T09:56:11,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971287) roll requested 2024-11-26T09:56:11,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971294 2024-11-26T09:56:11,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971294 2024-11-26T09:56:11,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971294) roll requested 2024-11-26T09:56:11,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971300 2024-11-26T09:56:11,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971300 2024-11-26T09:56:11,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971300) roll requested 2024-11-26T09:56:11,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971307 2024-11-26T09:56:11,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971307 2024-11-26T09:56:11,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971307) roll requested 2024-11-26T09:56:11,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971313 2024-11-26T09:56:11,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971313 2024-11-26T09:56:11,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971313) roll requested 2024-11-26T09:56:11,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971319 2024-11-26T09:56:11,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971319 2024-11-26T09:56:11,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971319) roll requested 2024-11-26T09:56:11,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971326 2024-11-26T09:56:11,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971326 2024-11-26T09:56:11,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971326) roll requested 2024-11-26T09:56:11,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971333 2024-11-26T09:56:11,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971333 2024-11-26T09:56:11,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971333) roll requested 2024-11-26T09:56:11,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971340 2024-11-26T09:56:11,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971340 2024-11-26T09:56:11,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971340) roll requested 2024-11-26T09:56:11,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971347 2024-11-26T09:56:11,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971347 2024-11-26T09:56:11,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971347) roll requested 2024-11-26T09:56:11,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971353 2024-11-26T09:56:11,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971353 2024-11-26T09:56:11,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971353) roll requested 2024-11-26T09:56:11,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971360 2024-11-26T09:56:11,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971360 2024-11-26T09:56:11,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971360) roll requested 2024-11-26T09:56:11,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971367 2024-11-26T09:56:11,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971367 2024-11-26T09:56:11,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971367) roll requested 2024-11-26T09:56:11,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971373 2024-11-26T09:56:11,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971373 2024-11-26T09:56:11,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971373) roll requested 2024-11-26T09:56:11,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971380 2024-11-26T09:56:11,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971380 2024-11-26T09:56:11,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971380) roll requested 2024-11-26T09:56:11,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971387 2024-11-26T09:56:11,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971387 2024-11-26T09:56:11,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971387) roll requested 2024-11-26T09:56:11,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971393 2024-11-26T09:56:11,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971393 2024-11-26T09:56:11,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971393) roll requested 2024-11-26T09:56:11,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971400 2024-11-26T09:56:11,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971400 2024-11-26T09:56:11,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971400) roll requested 2024-11-26T09:56:11,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971406 2024-11-26T09:56:11,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971406 2024-11-26T09:56:11,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971406) roll requested 2024-11-26T09:56:11,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971415 2024-11-26T09:56:11,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971415 2024-11-26T09:56:11,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971415) roll requested 2024-11-26T09:56:11,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971421 2024-11-26T09:56:11,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971421 2024-11-26T09:56:11,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971421) roll requested 2024-11-26T09:56:11,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971427 2024-11-26T09:56:11,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971427 2024-11-26T09:56:11,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971427) roll requested 2024-11-26T09:56:11,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971434 2024-11-26T09:56:11,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971434 2024-11-26T09:56:11,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971434) roll requested 2024-11-26T09:56:11,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971440 2024-11-26T09:56:11,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971440 2024-11-26T09:56:11,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971440) roll requested 2024-11-26T09:56:11,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971447 2024-11-26T09:56:11,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971447 2024-11-26T09:56:11,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971447) roll requested 2024-11-26T09:56:11,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971456 2024-11-26T09:56:11,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971456 2024-11-26T09:56:11,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971456) roll requested 2024-11-26T09:56:11,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971462 2024-11-26T09:56:11,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971462 2024-11-26T09:56:11,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971462) roll requested 2024-11-26T09:56:11,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971468 2024-11-26T09:56:11,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971468 2024-11-26T09:56:11,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971468) roll requested 2024-11-26T09:56:11,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971475 2024-11-26T09:56:11,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971475 2024-11-26T09:56:11,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971475) roll requested 2024-11-26T09:56:11,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971481 2024-11-26T09:56:11,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971481 2024-11-26T09:56:11,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971481) roll requested 2024-11-26T09:56:11,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971487 2024-11-26T09:56:11,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971487 2024-11-26T09:56:11,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971487) roll requested 2024-11-26T09:56:11,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971493 2024-11-26T09:56:11,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971493 2024-11-26T09:56:11,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971493) roll requested 2024-11-26T09:56:11,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971501 2024-11-26T09:56:11,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971501 2024-11-26T09:56:11,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971501) roll requested 2024-11-26T09:56:11,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971507 2024-11-26T09:56:11,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971507 2024-11-26T09:56:11,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971507) roll requested 2024-11-26T09:56:11,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971513 2024-11-26T09:56:11,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971513 2024-11-26T09:56:11,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971513) roll requested 2024-11-26T09:56:11,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971519 2024-11-26T09:56:11,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971519 2024-11-26T09:56:11,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971519) roll requested 2024-11-26T09:56:11,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971526 2024-11-26T09:56:11,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971526 2024-11-26T09:56:11,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971526) roll requested 2024-11-26T09:56:11,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971533 2024-11-26T09:56:11,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971533 2024-11-26T09:56:11,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971533) roll requested 2024-11-26T09:56:11,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971539 2024-11-26T09:56:11,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971539 2024-11-26T09:56:11,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971539) roll requested 2024-11-26T09:56:11,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971546 2024-11-26T09:56:11,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971546 2024-11-26T09:56:11,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971546) roll requested 2024-11-26T09:56:11,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971553 2024-11-26T09:56:11,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971553 2024-11-26T09:56:11,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971553) roll requested 2024-11-26T09:56:11,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971559 2024-11-26T09:56:11,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971559 2024-11-26T09:56:11,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971559) roll requested 2024-11-26T09:56:11,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971566 2024-11-26T09:56:11,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971566 2024-11-26T09:56:11,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971566) roll requested 2024-11-26T09:56:11,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971572 2024-11-26T09:56:11,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971572 2024-11-26T09:56:11,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971572) roll requested 2024-11-26T09:56:11,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971578 2024-11-26T09:56:11,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971578 2024-11-26T09:56:11,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971578) roll requested 2024-11-26T09:56:11,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971585 2024-11-26T09:56:11,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971585 2024-11-26T09:56:11,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971585) roll requested 2024-11-26T09:56:11,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971591 2024-11-26T09:56:11,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971591 2024-11-26T09:56:11,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971591) roll requested 2024-11-26T09:56:11,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971598 2024-11-26T09:56:11,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971598 2024-11-26T09:56:11,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971598) roll requested 2024-11-26T09:56:11,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971604 2024-11-26T09:56:11,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971604 2024-11-26T09:56:11,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971604) roll requested 2024-11-26T09:56:11,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971610 2024-11-26T09:56:11,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971610 2024-11-26T09:56:11,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971610) roll requested 2024-11-26T09:56:11,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971618 2024-11-26T09:56:11,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971618 2024-11-26T09:56:11,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971618) roll requested 2024-11-26T09:56:11,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971627 2024-11-26T09:56:11,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971627 2024-11-26T09:56:11,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971627) roll requested 2024-11-26T09:56:11,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971644 2024-11-26T09:56:11,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971644 2024-11-26T09:56:11,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971644) roll requested 2024-11-26T09:56:11,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971651 2024-11-26T09:56:11,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971651 2024-11-26T09:56:11,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971651) roll requested 2024-11-26T09:56:11,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971659 2024-11-26T09:56:11,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971659 2024-11-26T09:56:11,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971659) roll requested 2024-11-26T09:56:11,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971666 2024-11-26T09:56:11,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971666 2024-11-26T09:56:11,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971666) roll requested 2024-11-26T09:56:11,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971680 2024-11-26T09:56:11,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971680 2024-11-26T09:56:11,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971680) roll requested 2024-11-26T09:56:11,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971692 2024-11-26T09:56:11,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971692 2024-11-26T09:56:11,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971692) roll requested 2024-11-26T09:56:11,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971714 2024-11-26T09:56:11,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971714 2024-11-26T09:56:11,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971714) roll requested 2024-11-26T09:56:11,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971721 2024-11-26T09:56:11,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971721 2024-11-26T09:56:11,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971721) roll requested 2024-11-26T09:56:11,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971732 2024-11-26T09:56:11,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971732 2024-11-26T09:56:11,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971732) roll requested 2024-11-26T09:56:11,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971740 2024-11-26T09:56:11,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971740 2024-11-26T09:56:11,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971740) roll requested 2024-11-26T09:56:11,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971748 2024-11-26T09:56:11,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971748 2024-11-26T09:56:11,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971748) roll requested 2024-11-26T09:56:11,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971755 2024-11-26T09:56:11,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971755 2024-11-26T09:56:11,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971755) roll requested 2024-11-26T09:56:11,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971762 2024-11-26T09:56:11,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971762 2024-11-26T09:56:11,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971762) roll requested 2024-11-26T09:56:11,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971769 2024-11-26T09:56:11,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971769 2024-11-26T09:56:11,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971769) roll requested 2024-11-26T09:56:11,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971780 2024-11-26T09:56:11,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971780 2024-11-26T09:56:11,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971780) roll requested 2024-11-26T09:56:11,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971787 2024-11-26T09:56:11,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971787 2024-11-26T09:56:11,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971787) roll requested 2024-11-26T09:56:11,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971793 2024-11-26T09:56:11,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971793 2024-11-26T09:56:11,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971793) roll requested 2024-11-26T09:56:11,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971802 2024-11-26T09:56:11,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971802 2024-11-26T09:56:11,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971802) roll requested 2024-11-26T09:56:11,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971810 2024-11-26T09:56:11,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971810 2024-11-26T09:56:11,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971810) roll requested 2024-11-26T09:56:11,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971817 2024-11-26T09:56:11,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971817 2024-11-26T09:56:11,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971817) roll requested 2024-11-26T09:56:11,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971823 2024-11-26T09:56:11,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971823 2024-11-26T09:56:11,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971823) roll requested 2024-11-26T09:56:11,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971831 2024-11-26T09:56:11,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971831 2024-11-26T09:56:11,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971831) roll requested 2024-11-26T09:56:11,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971840 2024-11-26T09:56:11,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971840 2024-11-26T09:56:11,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971840) roll requested 2024-11-26T09:56:11,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971847 2024-11-26T09:56:11,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971847 2024-11-26T09:56:11,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971847) roll requested 2024-11-26T09:56:11,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971854 2024-11-26T09:56:11,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971854 2024-11-26T09:56:11,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971854) roll requested 2024-11-26T09:56:11,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971861 2024-11-26T09:56:11,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971861 2024-11-26T09:56:11,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971861) roll requested 2024-11-26T09:56:11,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971868 2024-11-26T09:56:11,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971868 2024-11-26T09:56:11,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971868) roll requested 2024-11-26T09:56:11,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971874 2024-11-26T09:56:11,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971874 2024-11-26T09:56:11,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971874) roll requested 2024-11-26T09:56:11,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971880 2024-11-26T09:56:11,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971880 2024-11-26T09:56:11,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971880) roll requested 2024-11-26T09:56:11,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971887 2024-11-26T09:56:11,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971887 2024-11-26T09:56:11,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971887) roll requested 2024-11-26T09:56:11,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971893 2024-11-26T09:56:11,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971893 2024-11-26T09:56:11,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971893) roll requested 2024-11-26T09:56:11,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971900 2024-11-26T09:56:11,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971900 2024-11-26T09:56:11,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971900) roll requested 2024-11-26T09:56:11,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971906 2024-11-26T09:56:11,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971906 2024-11-26T09:56:11,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971906) roll requested 2024-11-26T09:56:11,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971913 2024-11-26T09:56:11,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971913 2024-11-26T09:56:11,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971913) roll requested 2024-11-26T09:56:11,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971919 2024-11-26T09:56:11,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971919 2024-11-26T09:56:11,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971919) roll requested 2024-11-26T09:56:11,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971925 2024-11-26T09:56:11,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971925 2024-11-26T09:56:11,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971925) roll requested 2024-11-26T09:56:11,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971931 2024-11-26T09:56:11,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971931 2024-11-26T09:56:11,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971931) roll requested 2024-11-26T09:56:11,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971937 2024-11-26T09:56:11,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971937 2024-11-26T09:56:11,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971937) roll requested 2024-11-26T09:56:11,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971944 2024-11-26T09:56:11,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971944 2024-11-26T09:56:11,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971944) roll requested 2024-11-26T09:56:11,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971950 2024-11-26T09:56:11,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971950 2024-11-26T09:56:11,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971950) roll requested 2024-11-26T09:56:11,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971956 2024-11-26T09:56:11,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971956 2024-11-26T09:56:11,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971956) roll requested 2024-11-26T09:56:11,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971962 2024-11-26T09:56:11,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971962 2024-11-26T09:56:11,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971962) roll requested 2024-11-26T09:56:11,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971969 2024-11-26T09:56:11,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971969 2024-11-26T09:56:11,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971969) roll requested 2024-11-26T09:56:11,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971976 2024-11-26T09:56:11,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971976 2024-11-26T09:56:11,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971976) roll requested 2024-11-26T09:56:11,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971982 2024-11-26T09:56:11,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971982 2024-11-26T09:56:11,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971982) roll requested 2024-11-26T09:56:11,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971989 2024-11-26T09:56:11,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:11,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971989 2024-11-26T09:56:11,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:11,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:11,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971989) roll requested 2024-11-26T09:56:11,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614971995 2024-11-26T09:56:12,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971995 2024-11-26T09:56:12,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614971995) roll requested 2024-11-26T09:56:12,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972003 2024-11-26T09:56:12,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614971995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972003 2024-11-26T09:56:12,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972003) roll requested 2024-11-26T09:56:12,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972009 2024-11-26T09:56:12,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972009 2024-11-26T09:56:12,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972009) roll requested 2024-11-26T09:56:12,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972019 2024-11-26T09:56:12,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972019 2024-11-26T09:56:12,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972019) roll requested 2024-11-26T09:56:12,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972028 2024-11-26T09:56:12,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972028 2024-11-26T09:56:12,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972028) roll requested 2024-11-26T09:56:12,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972034 2024-11-26T09:56:12,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972034 2024-11-26T09:56:12,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972034) roll requested 2024-11-26T09:56:12,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972041 2024-11-26T09:56:12,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972041 2024-11-26T09:56:12,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972041) roll requested 2024-11-26T09:56:12,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972047 2024-11-26T09:56:12,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972047 2024-11-26T09:56:12,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972047) roll requested 2024-11-26T09:56:12,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972053 2024-11-26T09:56:12,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972053 2024-11-26T09:56:12,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972053) roll requested 2024-11-26T09:56:12,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972059 2024-11-26T09:56:12,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972059 2024-11-26T09:56:12,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972059) roll requested 2024-11-26T09:56:12,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972066 2024-11-26T09:56:12,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972066 2024-11-26T09:56:12,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972066) roll requested 2024-11-26T09:56:12,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972072 2024-11-26T09:56:12,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972072 2024-11-26T09:56:12,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972072) roll requested 2024-11-26T09:56:12,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972078 2024-11-26T09:56:12,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972078 2024-11-26T09:56:12,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972078) roll requested 2024-11-26T09:56:12,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972085 2024-11-26T09:56:12,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972085 2024-11-26T09:56:12,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972085) roll requested 2024-11-26T09:56:12,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972091 2024-11-26T09:56:12,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972091 2024-11-26T09:56:12,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972091) roll requested 2024-11-26T09:56:12,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972097 2024-11-26T09:56:12,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972097 2024-11-26T09:56:12,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972097) roll requested 2024-11-26T09:56:12,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972103 2024-11-26T09:56:12,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972103 2024-11-26T09:56:12,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972103) roll requested 2024-11-26T09:56:12,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972109 2024-11-26T09:56:12,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972109 2024-11-26T09:56:12,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972109) roll requested 2024-11-26T09:56:12,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972116 2024-11-26T09:56:12,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972116 2024-11-26T09:56:12,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972116) roll requested 2024-11-26T09:56:12,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972122 2024-11-26T09:56:12,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972122 2024-11-26T09:56:12,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972122) roll requested 2024-11-26T09:56:12,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972128 2024-11-26T09:56:12,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972128 2024-11-26T09:56:12,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972128) roll requested 2024-11-26T09:56:12,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972135 2024-11-26T09:56:12,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972135 2024-11-26T09:56:12,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972135) roll requested 2024-11-26T09:56:12,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972141 2024-11-26T09:56:12,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972141 2024-11-26T09:56:12,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972141) roll requested 2024-11-26T09:56:12,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972147 2024-11-26T09:56:12,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972147 2024-11-26T09:56:12,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972147) roll requested 2024-11-26T09:56:12,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972153 2024-11-26T09:56:12,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972153 2024-11-26T09:56:12,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972153) roll requested 2024-11-26T09:56:12,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972159 2024-11-26T09:56:12,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972159 2024-11-26T09:56:12,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972159) roll requested 2024-11-26T09:56:12,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972166 2024-11-26T09:56:12,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972166 2024-11-26T09:56:12,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972166) roll requested 2024-11-26T09:56:12,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972172 2024-11-26T09:56:12,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972172 2024-11-26T09:56:12,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972172) roll requested 2024-11-26T09:56:12,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972178 2024-11-26T09:56:12,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972178 2024-11-26T09:56:12,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972178) roll requested 2024-11-26T09:56:12,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972184 2024-11-26T09:56:12,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972184 2024-11-26T09:56:12,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972184) roll requested 2024-11-26T09:56:12,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972191 2024-11-26T09:56:12,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972191 2024-11-26T09:56:12,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972191) roll requested 2024-11-26T09:56:12,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972197 2024-11-26T09:56:12,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972197 2024-11-26T09:56:12,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972197) roll requested 2024-11-26T09:56:12,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972203 2024-11-26T09:56:12,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972203 2024-11-26T09:56:12,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972203) roll requested 2024-11-26T09:56:12,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972210 2024-11-26T09:56:12,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972210 2024-11-26T09:56:12,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972210) roll requested 2024-11-26T09:56:12,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972216 2024-11-26T09:56:12,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972216 2024-11-26T09:56:12,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972216) roll requested 2024-11-26T09:56:12,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972222 2024-11-26T09:56:12,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972222 2024-11-26T09:56:12,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972222) roll requested 2024-11-26T09:56:12,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972228 2024-11-26T09:56:12,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972228 2024-11-26T09:56:12,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972228) roll requested 2024-11-26T09:56:12,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972235 2024-11-26T09:56:12,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972235 2024-11-26T09:56:12,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972235) roll requested 2024-11-26T09:56:12,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972241 2024-11-26T09:56:12,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972241 2024-11-26T09:56:12,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972241) roll requested 2024-11-26T09:56:12,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972247 2024-11-26T09:56:12,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972247 2024-11-26T09:56:12,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972247) roll requested 2024-11-26T09:56:12,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972253 2024-11-26T09:56:12,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972253 2024-11-26T09:56:12,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972253) roll requested 2024-11-26T09:56:12,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972260 2024-11-26T09:56:12,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972260 2024-11-26T09:56:12,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972260) roll requested 2024-11-26T09:56:12,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972266 2024-11-26T09:56:12,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972266 2024-11-26T09:56:12,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972266) roll requested 2024-11-26T09:56:12,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972273 2024-11-26T09:56:12,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972273 2024-11-26T09:56:12,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972273) roll requested 2024-11-26T09:56:12,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972279 2024-11-26T09:56:12,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972279 2024-11-26T09:56:12,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972279) roll requested 2024-11-26T09:56:12,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972286 2024-11-26T09:56:12,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972286 2024-11-26T09:56:12,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972286) roll requested 2024-11-26T09:56:12,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972292 2024-11-26T09:56:12,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972292 2024-11-26T09:56:12,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972292) roll requested 2024-11-26T09:56:12,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972298 2024-11-26T09:56:12,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972298 2024-11-26T09:56:12,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972298) roll requested 2024-11-26T09:56:12,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972304 2024-11-26T09:56:12,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972304 2024-11-26T09:56:12,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972304) roll requested 2024-11-26T09:56:12,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972310 2024-11-26T09:56:12,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972310 2024-11-26T09:56:12,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972310) roll requested 2024-11-26T09:56:12,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972317 2024-11-26T09:56:12,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972317 2024-11-26T09:56:12,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972317) roll requested 2024-11-26T09:56:12,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972323 2024-11-26T09:56:12,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972323 2024-11-26T09:56:12,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972323) roll requested 2024-11-26T09:56:12,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972331 2024-11-26T09:56:12,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972331 2024-11-26T09:56:12,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972331) roll requested 2024-11-26T09:56:12,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972338 2024-11-26T09:56:12,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972338 2024-11-26T09:56:12,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972338) roll requested 2024-11-26T09:56:12,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972344 2024-11-26T09:56:12,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972344 2024-11-26T09:56:12,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972344) roll requested 2024-11-26T09:56:12,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972350 2024-11-26T09:56:12,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972350 2024-11-26T09:56:12,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972350) roll requested 2024-11-26T09:56:12,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972356 2024-11-26T09:56:12,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972356 2024-11-26T09:56:12,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972356) roll requested 2024-11-26T09:56:12,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972363 2024-11-26T09:56:12,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972363 2024-11-26T09:56:12,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972363) roll requested 2024-11-26T09:56:12,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972369 2024-11-26T09:56:12,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972369 2024-11-26T09:56:12,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972369) roll requested 2024-11-26T09:56:12,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972375 2024-11-26T09:56:12,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972375 2024-11-26T09:56:12,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972375) roll requested 2024-11-26T09:56:12,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972383 2024-11-26T09:56:12,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972383 2024-11-26T09:56:12,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972383) roll requested 2024-11-26T09:56:12,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972389 2024-11-26T09:56:12,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972389 2024-11-26T09:56:12,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972389) roll requested 2024-11-26T09:56:12,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972396 2024-11-26T09:56:12,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972396 2024-11-26T09:56:12,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972396) roll requested 2024-11-26T09:56:12,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972402 2024-11-26T09:56:12,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972402 2024-11-26T09:56:12,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972402) roll requested 2024-11-26T09:56:12,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972409 2024-11-26T09:56:12,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972409 2024-11-26T09:56:12,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972409) roll requested 2024-11-26T09:56:12,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972415 2024-11-26T09:56:12,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972415 2024-11-26T09:56:12,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972415) roll requested 2024-11-26T09:56:12,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972422 2024-11-26T09:56:12,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972422 2024-11-26T09:56:12,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972422) roll requested 2024-11-26T09:56:12,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972429 2024-11-26T09:56:12,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972429 2024-11-26T09:56:12,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972429) roll requested 2024-11-26T09:56:12,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972436 2024-11-26T09:56:12,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972436 2024-11-26T09:56:12,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972436) roll requested 2024-11-26T09:56:12,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972444 2024-11-26T09:56:12,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972444 2024-11-26T09:56:12,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972444) roll requested 2024-11-26T09:56:12,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972450 2024-11-26T09:56:12,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972450 2024-11-26T09:56:12,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972450) roll requested 2024-11-26T09:56:12,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972457 2024-11-26T09:56:12,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972457 2024-11-26T09:56:12,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972457) roll requested 2024-11-26T09:56:12,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972463 2024-11-26T09:56:12,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972463 2024-11-26T09:56:12,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972463) roll requested 2024-11-26T09:56:12,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972473 2024-11-26T09:56:12,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972473 2024-11-26T09:56:12,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972473) roll requested 2024-11-26T09:56:12,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972483 2024-11-26T09:56:12,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972483 2024-11-26T09:56:12,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972483) roll requested 2024-11-26T09:56:12,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972491 2024-11-26T09:56:12,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972491 2024-11-26T09:56:12,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972491) roll requested 2024-11-26T09:56:12,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972497 2024-11-26T09:56:12,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972497 2024-11-26T09:56:12,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972497) roll requested 2024-11-26T09:56:12,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972503 2024-11-26T09:56:12,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972503 2024-11-26T09:56:12,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972503) roll requested 2024-11-26T09:56:12,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972510 2024-11-26T09:56:12,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972510 2024-11-26T09:56:12,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972510) roll requested 2024-11-26T09:56:12,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972516 2024-11-26T09:56:12,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972516 2024-11-26T09:56:12,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972516) roll requested 2024-11-26T09:56:12,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972522 2024-11-26T09:56:12,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972522 2024-11-26T09:56:12,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972522) roll requested 2024-11-26T09:56:12,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972529 2024-11-26T09:56:12,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972529 2024-11-26T09:56:12,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972529) roll requested 2024-11-26T09:56:12,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972535 2024-11-26T09:56:12,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972535 2024-11-26T09:56:12,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972535) roll requested 2024-11-26T09:56:12,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972541 2024-11-26T09:56:12,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972541 2024-11-26T09:56:12,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972541) roll requested 2024-11-26T09:56:12,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972547 2024-11-26T09:56:12,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972547 2024-11-26T09:56:12,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972547) roll requested 2024-11-26T09:56:12,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972554 2024-11-26T09:56:12,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972554 2024-11-26T09:56:12,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972554) roll requested 2024-11-26T09:56:12,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972560 2024-11-26T09:56:12,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972560 2024-11-26T09:56:12,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972560) roll requested 2024-11-26T09:56:12,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972566 2024-11-26T09:56:12,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972566 2024-11-26T09:56:12,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972566) roll requested 2024-11-26T09:56:12,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972572 2024-11-26T09:56:12,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972572 2024-11-26T09:56:12,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972572) roll requested 2024-11-26T09:56:12,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972579 2024-11-26T09:56:12,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972579 2024-11-26T09:56:12,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972579) roll requested 2024-11-26T09:56:12,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972586 2024-11-26T09:56:12,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972586 2024-11-26T09:56:12,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972586) roll requested 2024-11-26T09:56:12,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972592 2024-11-26T09:56:12,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972592 2024-11-26T09:56:12,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972592) roll requested 2024-11-26T09:56:12,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972599 2024-11-26T09:56:12,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972599 2024-11-26T09:56:12,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972599) roll requested 2024-11-26T09:56:12,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972605 2024-11-26T09:56:12,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972605 2024-11-26T09:56:12,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972605) roll requested 2024-11-26T09:56:12,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972612 2024-11-26T09:56:12,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972612 2024-11-26T09:56:12,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972612) roll requested 2024-11-26T09:56:12,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972618 2024-11-26T09:56:12,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972618 2024-11-26T09:56:12,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972618) roll requested 2024-11-26T09:56:12,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972624 2024-11-26T09:56:12,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972624 2024-11-26T09:56:12,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972624) roll requested 2024-11-26T09:56:12,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972631 2024-11-26T09:56:12,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972631 2024-11-26T09:56:12,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972631) roll requested 2024-11-26T09:56:12,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972638 2024-11-26T09:56:12,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972638 2024-11-26T09:56:12,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972638) roll requested 2024-11-26T09:56:12,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972645 2024-11-26T09:56:12,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972645 2024-11-26T09:56:12,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972645) roll requested 2024-11-26T09:56:12,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972651 2024-11-26T09:56:12,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972651 2024-11-26T09:56:12,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972651) roll requested 2024-11-26T09:56:12,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972658 2024-11-26T09:56:12,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972658 2024-11-26T09:56:12,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972658) roll requested 2024-11-26T09:56:12,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972664 2024-11-26T09:56:12,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972664 2024-11-26T09:56:12,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972664) roll requested 2024-11-26T09:56:12,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972670 2024-11-26T09:56:12,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972670 2024-11-26T09:56:12,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972670) roll requested 2024-11-26T09:56:12,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972676 2024-11-26T09:56:12,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972676 2024-11-26T09:56:12,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972676) roll requested 2024-11-26T09:56:12,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972682 2024-11-26T09:56:12,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972682 2024-11-26T09:56:12,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972682) roll requested 2024-11-26T09:56:12,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972689 2024-11-26T09:56:12,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972689 2024-11-26T09:56:12,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972689) roll requested 2024-11-26T09:56:12,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972695 2024-11-26T09:56:12,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972695 2024-11-26T09:56:12,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972695) roll requested 2024-11-26T09:56:12,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972701 2024-11-26T09:56:12,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972701 2024-11-26T09:56:12,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972701) roll requested 2024-11-26T09:56:12,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972708 2024-11-26T09:56:12,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972708 2024-11-26T09:56:12,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972708) roll requested 2024-11-26T09:56:12,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972714 2024-11-26T09:56:12,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972714 2024-11-26T09:56:12,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972714) roll requested 2024-11-26T09:56:12,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972720 2024-11-26T09:56:12,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972720 2024-11-26T09:56:12,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972720) roll requested 2024-11-26T09:56:12,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972726 2024-11-26T09:56:12,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972726 2024-11-26T09:56:12,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972726) roll requested 2024-11-26T09:56:12,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972732 2024-11-26T09:56:12,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972732 2024-11-26T09:56:12,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,738 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972732) roll requested 2024-11-26T09:56:12,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972738 2024-11-26T09:56:12,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972738 2024-11-26T09:56:12,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972738) roll requested 2024-11-26T09:56:12,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972744 2024-11-26T09:56:12,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972744 2024-11-26T09:56:12,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972744) roll requested 2024-11-26T09:56:12,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972750 2024-11-26T09:56:12,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972750 2024-11-26T09:56:12,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972750) roll requested 2024-11-26T09:56:12,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972757 2024-11-26T09:56:12,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972757 2024-11-26T09:56:12,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972757) roll requested 2024-11-26T09:56:12,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972763 2024-11-26T09:56:12,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972763 2024-11-26T09:56:12,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972763) roll requested 2024-11-26T09:56:12,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972769 2024-11-26T09:56:12,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972769 2024-11-26T09:56:12,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972769) roll requested 2024-11-26T09:56:12,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972775 2024-11-26T09:56:12,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972775 2024-11-26T09:56:12,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972775) roll requested 2024-11-26T09:56:12,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972782 2024-11-26T09:56:12,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972782 2024-11-26T09:56:12,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972782) roll requested 2024-11-26T09:56:12,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972788 2024-11-26T09:56:12,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972788 2024-11-26T09:56:12,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972788) roll requested 2024-11-26T09:56:12,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972794 2024-11-26T09:56:12,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972794 2024-11-26T09:56:12,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972794) roll requested 2024-11-26T09:56:12,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972801 2024-11-26T09:56:12,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972801 2024-11-26T09:56:12,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972801) roll requested 2024-11-26T09:56:12,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972807 2024-11-26T09:56:12,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972807 2024-11-26T09:56:12,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972807) roll requested 2024-11-26T09:56:12,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972813 2024-11-26T09:56:12,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972813 2024-11-26T09:56:12,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972813) roll requested 2024-11-26T09:56:12,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972820 2024-11-26T09:56:12,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972820 2024-11-26T09:56:12,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972820) roll requested 2024-11-26T09:56:12,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972827 2024-11-26T09:56:12,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972827 2024-11-26T09:56:12,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972827) roll requested 2024-11-26T09:56:12,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972835 2024-11-26T09:56:12,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972835 2024-11-26T09:56:12,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972835) roll requested 2024-11-26T09:56:12,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972842 2024-11-26T09:56:12,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972842 2024-11-26T09:56:12,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972842) roll requested 2024-11-26T09:56:12,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972850 2024-11-26T09:56:12,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972850 2024-11-26T09:56:12,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972850) roll requested 2024-11-26T09:56:12,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972860 2024-11-26T09:56:12,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972860 2024-11-26T09:56:12,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972860) roll requested 2024-11-26T09:56:12,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972867 2024-11-26T09:56:12,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972867 2024-11-26T09:56:12,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972867) roll requested 2024-11-26T09:56:12,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972874 2024-11-26T09:56:12,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972874 2024-11-26T09:56:12,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972874) roll requested 2024-11-26T09:56:12,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972883 2024-11-26T09:56:12,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972883 2024-11-26T09:56:12,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972883) roll requested 2024-11-26T09:56:12,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972902 2024-11-26T09:56:12,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972902 2024-11-26T09:56:12,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972902) roll requested 2024-11-26T09:56:12,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972912 2024-11-26T09:56:12,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972912 2024-11-26T09:56:12,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972912) roll requested 2024-11-26T09:56:12,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972919 2024-11-26T09:56:12,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972919 2024-11-26T09:56:12,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972919) roll requested 2024-11-26T09:56:12,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972925 2024-11-26T09:56:12,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972925 2024-11-26T09:56:12,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972925) roll requested 2024-11-26T09:56:12,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972931 2024-11-26T09:56:12,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972931 2024-11-26T09:56:12,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972931) roll requested 2024-11-26T09:56:12,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972938 2024-11-26T09:56:12,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972938 2024-11-26T09:56:12,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972938) roll requested 2024-11-26T09:56:12,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972944 2024-11-26T09:56:12,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972944 2024-11-26T09:56:12,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972944) roll requested 2024-11-26T09:56:12,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972950 2024-11-26T09:56:12,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972950 2024-11-26T09:56:12,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972950) roll requested 2024-11-26T09:56:12,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972957 2024-11-26T09:56:12,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972957 2024-11-26T09:56:12,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972957) roll requested 2024-11-26T09:56:12,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972963 2024-11-26T09:56:12,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972963 2024-11-26T09:56:12,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972963) roll requested 2024-11-26T09:56:12,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972969 2024-11-26T09:56:12,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972969 2024-11-26T09:56:12,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972969) roll requested 2024-11-26T09:56:12,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972977 2024-11-26T09:56:12,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972977 2024-11-26T09:56:12,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972977) roll requested 2024-11-26T09:56:12,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972985 2024-11-26T09:56:12,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:12,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972985 2024-11-26T09:56:12,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:12,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:12,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972985) roll requested 2024-11-26T09:56:12,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614972995 2024-11-26T09:56:13,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972995 2024-11-26T09:56:13,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614972995) roll requested 2024-11-26T09:56:13,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973003 2024-11-26T09:56:13,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614972995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973003 2024-11-26T09:56:13,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973003) roll requested 2024-11-26T09:56:13,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973010 2024-11-26T09:56:13,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973010 2024-11-26T09:56:13,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973010) roll requested 2024-11-26T09:56:13,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973017 2024-11-26T09:56:13,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973017 2024-11-26T09:56:13,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973017) roll requested 2024-11-26T09:56:13,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973023 2024-11-26T09:56:13,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973023 2024-11-26T09:56:13,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973023) roll requested 2024-11-26T09:56:13,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973029 2024-11-26T09:56:13,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973029 2024-11-26T09:56:13,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973029) roll requested 2024-11-26T09:56:13,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973036 2024-11-26T09:56:13,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973036 2024-11-26T09:56:13,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973036) roll requested 2024-11-26T09:56:13,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973043 2024-11-26T09:56:13,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973043 2024-11-26T09:56:13,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973043) roll requested 2024-11-26T09:56:13,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973049 2024-11-26T09:56:13,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973049 2024-11-26T09:56:13,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973049) roll requested 2024-11-26T09:56:13,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973056 2024-11-26T09:56:13,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973056 2024-11-26T09:56:13,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973056) roll requested 2024-11-26T09:56:13,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973064 2024-11-26T09:56:13,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973064 2024-11-26T09:56:13,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973064) roll requested 2024-11-26T09:56:13,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973070 2024-11-26T09:56:13,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973070 2024-11-26T09:56:13,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973070) roll requested 2024-11-26T09:56:13,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973076 2024-11-26T09:56:13,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973076 2024-11-26T09:56:13,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973076) roll requested 2024-11-26T09:56:13,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973084 2024-11-26T09:56:13,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973084 2024-11-26T09:56:13,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973084) roll requested 2024-11-26T09:56:13,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973091 2024-11-26T09:56:13,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973091 2024-11-26T09:56:13,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973091) roll requested 2024-11-26T09:56:13,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973097 2024-11-26T09:56:13,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973097 2024-11-26T09:56:13,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973097) roll requested 2024-11-26T09:56:13,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973104 2024-11-26T09:56:13,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973104 2024-11-26T09:56:13,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973104) roll requested 2024-11-26T09:56:13,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973111 2024-11-26T09:56:13,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973111 2024-11-26T09:56:13,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973111) roll requested 2024-11-26T09:56:13,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973118 2024-11-26T09:56:13,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973118 2024-11-26T09:56:13,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973118) roll requested 2024-11-26T09:56:13,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973124 2024-11-26T09:56:13,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973124 2024-11-26T09:56:13,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973124) roll requested 2024-11-26T09:56:13,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973131 2024-11-26T09:56:13,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973131 2024-11-26T09:56:13,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973131) roll requested 2024-11-26T09:56:13,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973137 2024-11-26T09:56:13,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973137 2024-11-26T09:56:13,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973137) roll requested 2024-11-26T09:56:13,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973143 2024-11-26T09:56:13,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973143 2024-11-26T09:56:13,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973143) roll requested 2024-11-26T09:56:13,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973150 2024-11-26T09:56:13,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973150 2024-11-26T09:56:13,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973150) roll requested 2024-11-26T09:56:13,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973156 2024-11-26T09:56:13,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973156 2024-11-26T09:56:13,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973156) roll requested 2024-11-26T09:56:13,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973162 2024-11-26T09:56:13,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973162 2024-11-26T09:56:13,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973162) roll requested 2024-11-26T09:56:13,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973168 2024-11-26T09:56:13,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973168 2024-11-26T09:56:13,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973168) roll requested 2024-11-26T09:56:13,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973175 2024-11-26T09:56:13,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973175 2024-11-26T09:56:13,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973175) roll requested 2024-11-26T09:56:13,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973181 2024-11-26T09:56:13,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973181 2024-11-26T09:56:13,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973181) roll requested 2024-11-26T09:56:13,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973187 2024-11-26T09:56:13,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973187 2024-11-26T09:56:13,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973187) roll requested 2024-11-26T09:56:13,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973193 2024-11-26T09:56:13,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973193 2024-11-26T09:56:13,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973193) roll requested 2024-11-26T09:56:13,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973199 2024-11-26T09:56:13,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973199 2024-11-26T09:56:13,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973199) roll requested 2024-11-26T09:56:13,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973205 2024-11-26T09:56:13,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973205 2024-11-26T09:56:13,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973205) roll requested 2024-11-26T09:56:13,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973212 2024-11-26T09:56:13,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973212 2024-11-26T09:56:13,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973212) roll requested 2024-11-26T09:56:13,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973218 2024-11-26T09:56:13,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973218 2024-11-26T09:56:13,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973218) roll requested 2024-11-26T09:56:13,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973224 2024-11-26T09:56:13,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973224 2024-11-26T09:56:13,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973224) roll requested 2024-11-26T09:56:13,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973230 2024-11-26T09:56:13,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973230 2024-11-26T09:56:13,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973230) roll requested 2024-11-26T09:56:13,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973236 2024-11-26T09:56:13,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973236 2024-11-26T09:56:13,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973236) roll requested 2024-11-26T09:56:13,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973243 2024-11-26T09:56:13,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973243 2024-11-26T09:56:13,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973243) roll requested 2024-11-26T09:56:13,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973249 2024-11-26T09:56:13,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973249 2024-11-26T09:56:13,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973249) roll requested 2024-11-26T09:56:13,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973255 2024-11-26T09:56:13,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973255 2024-11-26T09:56:13,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973255) roll requested 2024-11-26T09:56:13,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973261 2024-11-26T09:56:13,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973261 2024-11-26T09:56:13,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973261) roll requested 2024-11-26T09:56:13,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973267 2024-11-26T09:56:13,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973267 2024-11-26T09:56:13,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973267) roll requested 2024-11-26T09:56:13,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973274 2024-11-26T09:56:13,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973274 2024-11-26T09:56:13,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973274) roll requested 2024-11-26T09:56:13,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973280 2024-11-26T09:56:13,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973280 2024-11-26T09:56:13,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,291 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973280) roll requested 2024-11-26T09:56:13,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973291 2024-11-26T09:56:13,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973291 2024-11-26T09:56:13,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973291) roll requested 2024-11-26T09:56:13,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973299 2024-11-26T09:56:13,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973291 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973299 2024-11-26T09:56:13,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973299) roll requested 2024-11-26T09:56:13,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973306 2024-11-26T09:56:13,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973306 2024-11-26T09:56:13,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973306) roll requested 2024-11-26T09:56:13,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973312 2024-11-26T09:56:13,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973312 2024-11-26T09:56:13,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973312) roll requested 2024-11-26T09:56:13,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973324 2024-11-26T09:56:13,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973324 2024-11-26T09:56:13,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973324) roll requested 2024-11-26T09:56:13,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973330 2024-11-26T09:56:13,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973330 2024-11-26T09:56:13,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973330) roll requested 2024-11-26T09:56:13,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973339 2024-11-26T09:56:13,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973339 2024-11-26T09:56:13,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973339) roll requested 2024-11-26T09:56:13,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973348 2024-11-26T09:56:13,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973348 2024-11-26T09:56:13,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973348) roll requested 2024-11-26T09:56:13,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973354 2024-11-26T09:56:13,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973354 2024-11-26T09:56:13,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973354) roll requested 2024-11-26T09:56:13,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973362 2024-11-26T09:56:13,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973362 2024-11-26T09:56:13,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973362) roll requested 2024-11-26T09:56:13,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973368 2024-11-26T09:56:13,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973368 2024-11-26T09:56:13,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973368) roll requested 2024-11-26T09:56:13,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973375 2024-11-26T09:56:13,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973375 2024-11-26T09:56:13,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973375) roll requested 2024-11-26T09:56:13,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973383 2024-11-26T09:56:13,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973383 2024-11-26T09:56:13,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973383) roll requested 2024-11-26T09:56:13,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973392 2024-11-26T09:56:13,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973392 2024-11-26T09:56:13,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973392) roll requested 2024-11-26T09:56:13,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973400 2024-11-26T09:56:13,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973400 2024-11-26T09:56:13,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973400) roll requested 2024-11-26T09:56:13,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973408 2024-11-26T09:56:13,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973408 2024-11-26T09:56:13,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973408) roll requested 2024-11-26T09:56:13,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973414 2024-11-26T09:56:13,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973414 2024-11-26T09:56:13,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973414) roll requested 2024-11-26T09:56:13,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973421 2024-11-26T09:56:13,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973421 2024-11-26T09:56:13,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973421) roll requested 2024-11-26T09:56:13,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973427 2024-11-26T09:56:13,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973427 2024-11-26T09:56:13,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973427) roll requested 2024-11-26T09:56:13,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973434 2024-11-26T09:56:13,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973434 2024-11-26T09:56:13,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973434) roll requested 2024-11-26T09:56:13,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973440 2024-11-26T09:56:13,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973440 2024-11-26T09:56:13,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973440) roll requested 2024-11-26T09:56:13,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973447 2024-11-26T09:56:13,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973447 2024-11-26T09:56:13,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973447) roll requested 2024-11-26T09:56:13,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973453 2024-11-26T09:56:13,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973453 2024-11-26T09:56:13,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973453) roll requested 2024-11-26T09:56:13,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973459 2024-11-26T09:56:13,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973459 2024-11-26T09:56:13,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973459) roll requested 2024-11-26T09:56:13,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973466 2024-11-26T09:56:13,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973466 2024-11-26T09:56:13,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973466) roll requested 2024-11-26T09:56:13,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973472 2024-11-26T09:56:13,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973472 2024-11-26T09:56:13,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973472) roll requested 2024-11-26T09:56:13,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973478 2024-11-26T09:56:13,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973478 2024-11-26T09:56:13,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973478) roll requested 2024-11-26T09:56:13,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973485 2024-11-26T09:56:13,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973485 2024-11-26T09:56:13,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973485) roll requested 2024-11-26T09:56:13,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973491 2024-11-26T09:56:13,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973491 2024-11-26T09:56:13,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973491) roll requested 2024-11-26T09:56:13,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973497 2024-11-26T09:56:13,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973497 2024-11-26T09:56:13,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973497) roll requested 2024-11-26T09:56:13,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973503 2024-11-26T09:56:13,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973503 2024-11-26T09:56:13,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973503) roll requested 2024-11-26T09:56:13,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973510 2024-11-26T09:56:13,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973510 2024-11-26T09:56:13,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973510) roll requested 2024-11-26T09:56:13,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973516 2024-11-26T09:56:13,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973516 2024-11-26T09:56:13,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973516) roll requested 2024-11-26T09:56:13,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973522 2024-11-26T09:56:13,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973522 2024-11-26T09:56:13,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973522) roll requested 2024-11-26T09:56:13,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973528 2024-11-26T09:56:13,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973528 2024-11-26T09:56:13,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973528) roll requested 2024-11-26T09:56:13,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973534 2024-11-26T09:56:13,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973534 2024-11-26T09:56:13,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973534) roll requested 2024-11-26T09:56:13,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973541 2024-11-26T09:56:13,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973541 2024-11-26T09:56:13,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973541) roll requested 2024-11-26T09:56:13,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973547 2024-11-26T09:56:13,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973547 2024-11-26T09:56:13,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,554 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973547) roll requested 2024-11-26T09:56:13,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973554 2024-11-26T09:56:13,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973554 2024-11-26T09:56:13,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973554) roll requested 2024-11-26T09:56:13,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973560 2024-11-26T09:56:13,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973560 2024-11-26T09:56:13,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973560) roll requested 2024-11-26T09:56:13,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973567 2024-11-26T09:56:13,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973567 2024-11-26T09:56:13,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973567) roll requested 2024-11-26T09:56:13,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973574 2024-11-26T09:56:13,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973574 2024-11-26T09:56:13,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973574) roll requested 2024-11-26T09:56:13,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973581 2024-11-26T09:56:13,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973581 2024-11-26T09:56:13,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973581) roll requested 2024-11-26T09:56:13,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973591 2024-11-26T09:56:13,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973591 2024-11-26T09:56:13,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973591) roll requested 2024-11-26T09:56:13,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973604 2024-11-26T09:56:13,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973604 2024-11-26T09:56:13,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973604) roll requested 2024-11-26T09:56:13,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973612 2024-11-26T09:56:13,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973612 2024-11-26T09:56:13,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973612) roll requested 2024-11-26T09:56:13,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973620 2024-11-26T09:56:13,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973620 2024-11-26T09:56:13,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973620) roll requested 2024-11-26T09:56:13,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973628 2024-11-26T09:56:13,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973628 2024-11-26T09:56:13,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973628) roll requested 2024-11-26T09:56:13,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973635 2024-11-26T09:56:13,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973635 2024-11-26T09:56:13,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973635) roll requested 2024-11-26T09:56:13,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973642 2024-11-26T09:56:13,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973642 2024-11-26T09:56:13,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973642) roll requested 2024-11-26T09:56:13,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973649 2024-11-26T09:56:13,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973649 2024-11-26T09:56:13,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973649) roll requested 2024-11-26T09:56:13,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973658 2024-11-26T09:56:13,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973658 2024-11-26T09:56:13,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973658) roll requested 2024-11-26T09:56:13,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973664 2024-11-26T09:56:13,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973664 2024-11-26T09:56:13,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973664) roll requested 2024-11-26T09:56:13,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973670 2024-11-26T09:56:13,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973670 2024-11-26T09:56:13,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973670) roll requested 2024-11-26T09:56:13,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973677 2024-11-26T09:56:13,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973677 2024-11-26T09:56:13,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973677) roll requested 2024-11-26T09:56:13,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973683 2024-11-26T09:56:13,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973683 2024-11-26T09:56:13,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973683) roll requested 2024-11-26T09:56:13,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973689 2024-11-26T09:56:13,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973689 2024-11-26T09:56:13,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973689) roll requested 2024-11-26T09:56:13,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973699 2024-11-26T09:56:13,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973699 2024-11-26T09:56:13,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973699) roll requested 2024-11-26T09:56:13,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973705 2024-11-26T09:56:13,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973705 2024-11-26T09:56:13,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973705) roll requested 2024-11-26T09:56:13,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973713 2024-11-26T09:56:13,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973713 2024-11-26T09:56:13,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973713) roll requested 2024-11-26T09:56:13,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973719 2024-11-26T09:56:13,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973719 2024-11-26T09:56:13,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973719) roll requested 2024-11-26T09:56:13,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973726 2024-11-26T09:56:13,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973726 2024-11-26T09:56:13,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973726) roll requested 2024-11-26T09:56:13,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973733 2024-11-26T09:56:13,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973733 2024-11-26T09:56:13,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973733) roll requested 2024-11-26T09:56:13,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973740 2024-11-26T09:56:13,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973740 2024-11-26T09:56:13,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973740) roll requested 2024-11-26T09:56:13,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973747 2024-11-26T09:56:13,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973747 2024-11-26T09:56:13,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973747) roll requested 2024-11-26T09:56:13,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973754 2024-11-26T09:56:13,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973754 2024-11-26T09:56:13,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973754) roll requested 2024-11-26T09:56:13,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973760 2024-11-26T09:56:13,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973760 2024-11-26T09:56:13,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973760) roll requested 2024-11-26T09:56:13,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973767 2024-11-26T09:56:13,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973767 2024-11-26T09:56:13,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973767) roll requested 2024-11-26T09:56:13,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973776 2024-11-26T09:56:13,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973776 2024-11-26T09:56:13,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973776) roll requested 2024-11-26T09:56:13,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973782 2024-11-26T09:56:13,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973782 2024-11-26T09:56:13,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973782) roll requested 2024-11-26T09:56:13,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973789 2024-11-26T09:56:13,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973789 2024-11-26T09:56:13,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973789) roll requested 2024-11-26T09:56:13,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973795 2024-11-26T09:56:13,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973795 2024-11-26T09:56:13,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973795) roll requested 2024-11-26T09:56:13,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973801 2024-11-26T09:56:13,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973801 2024-11-26T09:56:13,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973801) roll requested 2024-11-26T09:56:13,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973807 2024-11-26T09:56:13,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973807 2024-11-26T09:56:13,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973807) roll requested 2024-11-26T09:56:13,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973814 2024-11-26T09:56:13,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973814 2024-11-26T09:56:13,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973814) roll requested 2024-11-26T09:56:13,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973820 2024-11-26T09:56:13,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973820 2024-11-26T09:56:13,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973820) roll requested 2024-11-26T09:56:13,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973826 2024-11-26T09:56:13,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973826 2024-11-26T09:56:13,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973826) roll requested 2024-11-26T09:56:13,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973833 2024-11-26T09:56:13,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973833 2024-11-26T09:56:13,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973833) roll requested 2024-11-26T09:56:13,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973839 2024-11-26T09:56:13,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973839 2024-11-26T09:56:13,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973839) roll requested 2024-11-26T09:56:13,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973845 2024-11-26T09:56:13,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973845 2024-11-26T09:56:13,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973845) roll requested 2024-11-26T09:56:13,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973851 2024-11-26T09:56:13,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973851 2024-11-26T09:56:13,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973851) roll requested 2024-11-26T09:56:13,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973857 2024-11-26T09:56:13,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973857 2024-11-26T09:56:13,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973857) roll requested 2024-11-26T09:56:13,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973863 2024-11-26T09:56:13,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973863 2024-11-26T09:56:13,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973863) roll requested 2024-11-26T09:56:13,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973870 2024-11-26T09:56:13,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973870 2024-11-26T09:56:13,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973870) roll requested 2024-11-26T09:56:13,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973876 2024-11-26T09:56:13,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973876 2024-11-26T09:56:13,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,882 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973876) roll requested 2024-11-26T09:56:13,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973882 2024-11-26T09:56:13,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973882 2024-11-26T09:56:13,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973882) roll requested 2024-11-26T09:56:13,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973889 2024-11-26T09:56:13,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973889 2024-11-26T09:56:13,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973889) roll requested 2024-11-26T09:56:13,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973896 2024-11-26T09:56:13,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973896 2024-11-26T09:56:13,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973896) roll requested 2024-11-26T09:56:13,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973902 2024-11-26T09:56:13,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973902 2024-11-26T09:56:13,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973902) roll requested 2024-11-26T09:56:13,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973908 2024-11-26T09:56:13,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973908 2024-11-26T09:56:13,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973908) roll requested 2024-11-26T09:56:13,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973916 2024-11-26T09:56:13,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973916 2024-11-26T09:56:13,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973916) roll requested 2024-11-26T09:56:13,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973922 2024-11-26T09:56:13,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973922 2024-11-26T09:56:13,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973922) roll requested 2024-11-26T09:56:13,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973928 2024-11-26T09:56:13,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973928 2024-11-26T09:56:13,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973928) roll requested 2024-11-26T09:56:13,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973934 2024-11-26T09:56:13,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973934 2024-11-26T09:56:13,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973934) roll requested 2024-11-26T09:56:13,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973941 2024-11-26T09:56:13,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973941 2024-11-26T09:56:13,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973941) roll requested 2024-11-26T09:56:13,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973947 2024-11-26T09:56:13,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973947 2024-11-26T09:56:13,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973947) roll requested 2024-11-26T09:56:13,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973953 2024-11-26T09:56:13,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973953 2024-11-26T09:56:13,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973953) roll requested 2024-11-26T09:56:13,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973959 2024-11-26T09:56:13,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973959 2024-11-26T09:56:13,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973959) roll requested 2024-11-26T09:56:13,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973966 2024-11-26T09:56:13,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973966 2024-11-26T09:56:13,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973966) roll requested 2024-11-26T09:56:13,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973972 2024-11-26T09:56:13,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973972 2024-11-26T09:56:13,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973972) roll requested 2024-11-26T09:56:13,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973978 2024-11-26T09:56:13,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973978 2024-11-26T09:56:13,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973978) roll requested 2024-11-26T09:56:13,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973984 2024-11-26T09:56:13,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973984 2024-11-26T09:56:13,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973984) roll requested 2024-11-26T09:56:13,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973991 2024-11-26T09:56:13,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:13,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973991 2024-11-26T09:56:13,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:13,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:13,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973991) roll requested 2024-11-26T09:56:13,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614973997 2024-11-26T09:56:14,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973997 2024-11-26T09:56:14,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614973997) roll requested 2024-11-26T09:56:14,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974003 2024-11-26T09:56:14,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614973997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974003 2024-11-26T09:56:14,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974003) roll requested 2024-11-26T09:56:14,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974010 2024-11-26T09:56:14,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974010 2024-11-26T09:56:14,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974010) roll requested 2024-11-26T09:56:14,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974016 2024-11-26T09:56:14,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974016 2024-11-26T09:56:14,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974016) roll requested 2024-11-26T09:56:14,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974022 2024-11-26T09:56:14,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974022 2024-11-26T09:56:14,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974022) roll requested 2024-11-26T09:56:14,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974028 2024-11-26T09:56:14,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974028 2024-11-26T09:56:14,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974028) roll requested 2024-11-26T09:56:14,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974035 2024-11-26T09:56:14,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974035 2024-11-26T09:56:14,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974035) roll requested 2024-11-26T09:56:14,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974042 2024-11-26T09:56:14,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974042 2024-11-26T09:56:14,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974042) roll requested 2024-11-26T09:56:14,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974056 2024-11-26T09:56:14,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974056 2024-11-26T09:56:14,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974056) roll requested 2024-11-26T09:56:14,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974063 2024-11-26T09:56:14,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974063 2024-11-26T09:56:14,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974063) roll requested 2024-11-26T09:56:14,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974069 2024-11-26T09:56:14,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974069 2024-11-26T09:56:14,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974069) roll requested 2024-11-26T09:56:14,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974075 2024-11-26T09:56:14,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974075 2024-11-26T09:56:14,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974075) roll requested 2024-11-26T09:56:14,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974081 2024-11-26T09:56:14,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974081 2024-11-26T09:56:14,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974081) roll requested 2024-11-26T09:56:14,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974088 2024-11-26T09:56:14,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974088 2024-11-26T09:56:14,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974088) roll requested 2024-11-26T09:56:14,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974100 2024-11-26T09:56:14,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974100 2024-11-26T09:56:14,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974100) roll requested 2024-11-26T09:56:14,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974107 2024-11-26T09:56:14,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974107 2024-11-26T09:56:14,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974107) roll requested 2024-11-26T09:56:14,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974116 2024-11-26T09:56:14,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974116 2024-11-26T09:56:14,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974116) roll requested 2024-11-26T09:56:14,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974123 2024-11-26T09:56:14,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974123 2024-11-26T09:56:14,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974123) roll requested 2024-11-26T09:56:14,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974129 2024-11-26T09:56:14,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974129 2024-11-26T09:56:14,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974129) roll requested 2024-11-26T09:56:14,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974135 2024-11-26T09:56:14,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974135 2024-11-26T09:56:14,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974135) roll requested 2024-11-26T09:56:14,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974142 2024-11-26T09:56:14,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974142 2024-11-26T09:56:14,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974142) roll requested 2024-11-26T09:56:14,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974148 2024-11-26T09:56:14,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974148 2024-11-26T09:56:14,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974148) roll requested 2024-11-26T09:56:14,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974154 2024-11-26T09:56:14,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974154 2024-11-26T09:56:14,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974154) roll requested 2024-11-26T09:56:14,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974161 2024-11-26T09:56:14,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974161 2024-11-26T09:56:14,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974161) roll requested 2024-11-26T09:56:14,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974168 2024-11-26T09:56:14,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974168 2024-11-26T09:56:14,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974168) roll requested 2024-11-26T09:56:14,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974174 2024-11-26T09:56:14,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974174 2024-11-26T09:56:14,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974174) roll requested 2024-11-26T09:56:14,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974180 2024-11-26T09:56:14,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974180 2024-11-26T09:56:14,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974180) roll requested 2024-11-26T09:56:14,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974188 2024-11-26T09:56:14,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974188 2024-11-26T09:56:14,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974188) roll requested 2024-11-26T09:56:14,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974194 2024-11-26T09:56:14,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974194 2024-11-26T09:56:14,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974194) roll requested 2024-11-26T09:56:14,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974200 2024-11-26T09:56:14,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974200 2024-11-26T09:56:14,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974200) roll requested 2024-11-26T09:56:14,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974206 2024-11-26T09:56:14,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974206 2024-11-26T09:56:14,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974206) roll requested 2024-11-26T09:56:14,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974213 2024-11-26T09:56:14,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974213 2024-11-26T09:56:14,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974213) roll requested 2024-11-26T09:56:14,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974219 2024-11-26T09:56:14,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974219 2024-11-26T09:56:14,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974219) roll requested 2024-11-26T09:56:14,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974225 2024-11-26T09:56:14,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974225 2024-11-26T09:56:14,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974225) roll requested 2024-11-26T09:56:14,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974231 2024-11-26T09:56:14,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974231 2024-11-26T09:56:14,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974231) roll requested 2024-11-26T09:56:14,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974238 2024-11-26T09:56:14,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974238 2024-11-26T09:56:14,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974238) roll requested 2024-11-26T09:56:14,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974244 2024-11-26T09:56:14,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974244 2024-11-26T09:56:14,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974244) roll requested 2024-11-26T09:56:14,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974250 2024-11-26T09:56:14,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974250 2024-11-26T09:56:14,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974250) roll requested 2024-11-26T09:56:14,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974257 2024-11-26T09:56:14,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974257 2024-11-26T09:56:14,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974257) roll requested 2024-11-26T09:56:14,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974263 2024-11-26T09:56:14,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974263 2024-11-26T09:56:14,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974263) roll requested 2024-11-26T09:56:14,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974269 2024-11-26T09:56:14,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974269 2024-11-26T09:56:14,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974269) roll requested 2024-11-26T09:56:14,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974275 2024-11-26T09:56:14,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974275 2024-11-26T09:56:14,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974275) roll requested 2024-11-26T09:56:14,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974282 2024-11-26T09:56:14,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974282 2024-11-26T09:56:14,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974282) roll requested 2024-11-26T09:56:14,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974288 2024-11-26T09:56:14,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974288 2024-11-26T09:56:14,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974288) roll requested 2024-11-26T09:56:14,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974294 2024-11-26T09:56:14,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974294 2024-11-26T09:56:14,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974294) roll requested 2024-11-26T09:56:14,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974301 2024-11-26T09:56:14,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974301 2024-11-26T09:56:14,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974301) roll requested 2024-11-26T09:56:14,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974307 2024-11-26T09:56:14,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974307 2024-11-26T09:56:14,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974307) roll requested 2024-11-26T09:56:14,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974313 2024-11-26T09:56:14,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974313 2024-11-26T09:56:14,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974313) roll requested 2024-11-26T09:56:14,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974319 2024-11-26T09:56:14,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974319 2024-11-26T09:56:14,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974319) roll requested 2024-11-26T09:56:14,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974326 2024-11-26T09:56:14,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974326 2024-11-26T09:56:14,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974326) roll requested 2024-11-26T09:56:14,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974332 2024-11-26T09:56:14,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974332 2024-11-26T09:56:14,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974332) roll requested 2024-11-26T09:56:14,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974338 2024-11-26T09:56:14,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974338 2024-11-26T09:56:14,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974338) roll requested 2024-11-26T09:56:14,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974344 2024-11-26T09:56:14,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974344 2024-11-26T09:56:14,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974344) roll requested 2024-11-26T09:56:14,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974350 2024-11-26T09:56:14,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974350 2024-11-26T09:56:14,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974350) roll requested 2024-11-26T09:56:14,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974356 2024-11-26T09:56:14,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974356 2024-11-26T09:56:14,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974356) roll requested 2024-11-26T09:56:14,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974363 2024-11-26T09:56:14,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974363 2024-11-26T09:56:14,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974363) roll requested 2024-11-26T09:56:14,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974369 2024-11-26T09:56:14,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974369 2024-11-26T09:56:14,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974369) roll requested 2024-11-26T09:56:14,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974375 2024-11-26T09:56:14,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974375 2024-11-26T09:56:14,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974375) roll requested 2024-11-26T09:56:14,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974381 2024-11-26T09:56:14,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974381 2024-11-26T09:56:14,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974381) roll requested 2024-11-26T09:56:14,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974388 2024-11-26T09:56:14,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974388 2024-11-26T09:56:14,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974388) roll requested 2024-11-26T09:56:14,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974394 2024-11-26T09:56:14,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974394 2024-11-26T09:56:14,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974394) roll requested 2024-11-26T09:56:14,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974400 2024-11-26T09:56:14,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974400 2024-11-26T09:56:14,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974400) roll requested 2024-11-26T09:56:14,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974406 2024-11-26T09:56:14,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974406 2024-11-26T09:56:14,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974406) roll requested 2024-11-26T09:56:14,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974413 2024-11-26T09:56:14,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974413 2024-11-26T09:56:14,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974413) roll requested 2024-11-26T09:56:14,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974419 2024-11-26T09:56:14,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974419 2024-11-26T09:56:14,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974419) roll requested 2024-11-26T09:56:14,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974426 2024-11-26T09:56:14,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974426 2024-11-26T09:56:14,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974426) roll requested 2024-11-26T09:56:14,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974432 2024-11-26T09:56:14,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974432 2024-11-26T09:56:14,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974432) roll requested 2024-11-26T09:56:14,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974438 2024-11-26T09:56:14,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974438 2024-11-26T09:56:14,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974438) roll requested 2024-11-26T09:56:14,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974445 2024-11-26T09:56:14,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974445 2024-11-26T09:56:14,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974445) roll requested 2024-11-26T09:56:14,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974451 2024-11-26T09:56:14,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974451 2024-11-26T09:56:14,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974451) roll requested 2024-11-26T09:56:14,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974458 2024-11-26T09:56:14,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974458 2024-11-26T09:56:14,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974458) roll requested 2024-11-26T09:56:14,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974464 2024-11-26T09:56:14,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974464 2024-11-26T09:56:14,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974464) roll requested 2024-11-26T09:56:14,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974470 2024-11-26T09:56:14,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974470 2024-11-26T09:56:14,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974470) roll requested 2024-11-26T09:56:14,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974476 2024-11-26T09:56:14,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974476 2024-11-26T09:56:14,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974476) roll requested 2024-11-26T09:56:14,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974483 2024-11-26T09:56:14,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974483 2024-11-26T09:56:14,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974483) roll requested 2024-11-26T09:56:14,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974489 2024-11-26T09:56:14,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974489 2024-11-26T09:56:14,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974489) roll requested 2024-11-26T09:56:14,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974495 2024-11-26T09:56:14,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974495 2024-11-26T09:56:14,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974495) roll requested 2024-11-26T09:56:14,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974502 2024-11-26T09:56:14,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974502 2024-11-26T09:56:14,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974502) roll requested 2024-11-26T09:56:14,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974508 2024-11-26T09:56:14,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974508 2024-11-26T09:56:14,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974508) roll requested 2024-11-26T09:56:14,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974514 2024-11-26T09:56:14,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974514 2024-11-26T09:56:14,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974514) roll requested 2024-11-26T09:56:14,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974521 2024-11-26T09:56:14,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974521 2024-11-26T09:56:14,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974521) roll requested 2024-11-26T09:56:14,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974527 2024-11-26T09:56:14,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974527 2024-11-26T09:56:14,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974527) roll requested 2024-11-26T09:56:14,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974533 2024-11-26T09:56:14,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974533 2024-11-26T09:56:14,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974533) roll requested 2024-11-26T09:56:14,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974539 2024-11-26T09:56:14,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974539 2024-11-26T09:56:14,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974539) roll requested 2024-11-26T09:56:14,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974545 2024-11-26T09:56:14,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974545 2024-11-26T09:56:14,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974545) roll requested 2024-11-26T09:56:14,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974552 2024-11-26T09:56:14,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974552 2024-11-26T09:56:14,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974552) roll requested 2024-11-26T09:56:14,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974558 2024-11-26T09:56:14,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974558 2024-11-26T09:56:14,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974558) roll requested 2024-11-26T09:56:14,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974564 2024-11-26T09:56:14,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974564 2024-11-26T09:56:14,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974564) roll requested 2024-11-26T09:56:14,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974571 2024-11-26T09:56:14,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974571 2024-11-26T09:56:14,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974571) roll requested 2024-11-26T09:56:14,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974577 2024-11-26T09:56:14,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974577 2024-11-26T09:56:14,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974577) roll requested 2024-11-26T09:56:14,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974583 2024-11-26T09:56:14,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974583 2024-11-26T09:56:14,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974583) roll requested 2024-11-26T09:56:14,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974589 2024-11-26T09:56:14,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974589 2024-11-26T09:56:14,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974589) roll requested 2024-11-26T09:56:14,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974596 2024-11-26T09:56:14,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974596 2024-11-26T09:56:14,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974596) roll requested 2024-11-26T09:56:14,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974602 2024-11-26T09:56:14,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974602 2024-11-26T09:56:14,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974602) roll requested 2024-11-26T09:56:14,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974608 2024-11-26T09:56:14,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974608 2024-11-26T09:56:14,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974608) roll requested 2024-11-26T09:56:14,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974615 2024-11-26T09:56:14,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974615 2024-11-26T09:56:14,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974615) roll requested 2024-11-26T09:56:14,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974622 2024-11-26T09:56:14,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974622 2024-11-26T09:56:14,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974622) roll requested 2024-11-26T09:56:14,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974628 2024-11-26T09:56:14,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974628 2024-11-26T09:56:14,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974628) roll requested 2024-11-26T09:56:14,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974635 2024-11-26T09:56:14,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974635 2024-11-26T09:56:14,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974635) roll requested 2024-11-26T09:56:14,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974641 2024-11-26T09:56:14,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974641 2024-11-26T09:56:14,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974641) roll requested 2024-11-26T09:56:14,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974647 2024-11-26T09:56:14,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974647 2024-11-26T09:56:14,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974647) roll requested 2024-11-26T09:56:14,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974654 2024-11-26T09:56:14,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974654 2024-11-26T09:56:14,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974654) roll requested 2024-11-26T09:56:14,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974660 2024-11-26T09:56:14,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974660 2024-11-26T09:56:14,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974660) roll requested 2024-11-26T09:56:14,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974666 2024-11-26T09:56:14,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974666 2024-11-26T09:56:14,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974666) roll requested 2024-11-26T09:56:14,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974672 2024-11-26T09:56:14,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974672 2024-11-26T09:56:14,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974672) roll requested 2024-11-26T09:56:14,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974678 2024-11-26T09:56:14,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974678 2024-11-26T09:56:14,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974678) roll requested 2024-11-26T09:56:14,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974685 2024-11-26T09:56:14,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974685 2024-11-26T09:56:14,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974685) roll requested 2024-11-26T09:56:14,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974691 2024-11-26T09:56:14,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974691 2024-11-26T09:56:14,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974691) roll requested 2024-11-26T09:56:14,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974697 2024-11-26T09:56:14,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974691 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974697 2024-11-26T09:56:14,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974697) roll requested 2024-11-26T09:56:14,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974703 2024-11-26T09:56:14,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974703 2024-11-26T09:56:14,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974703) roll requested 2024-11-26T09:56:14,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974709 2024-11-26T09:56:14,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974709 2024-11-26T09:56:14,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974709) roll requested 2024-11-26T09:56:14,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974715 2024-11-26T09:56:14,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974715 2024-11-26T09:56:14,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974715) roll requested 2024-11-26T09:56:14,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974721 2024-11-26T09:56:14,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974721 2024-11-26T09:56:14,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974721) roll requested 2024-11-26T09:56:14,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974727 2024-11-26T09:56:14,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974727 2024-11-26T09:56:14,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974727) roll requested 2024-11-26T09:56:14,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974733 2024-11-26T09:56:14,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974733 2024-11-26T09:56:14,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974733) roll requested 2024-11-26T09:56:14,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974742 2024-11-26T09:56:14,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974742 2024-11-26T09:56:14,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974742) roll requested 2024-11-26T09:56:14,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974749 2024-11-26T09:56:14,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974749 2024-11-26T09:56:14,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974749) roll requested 2024-11-26T09:56:14,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974755 2024-11-26T09:56:14,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974755 2024-11-26T09:56:14,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974755) roll requested 2024-11-26T09:56:14,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974761 2024-11-26T09:56:14,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974761 2024-11-26T09:56:14,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974761) roll requested 2024-11-26T09:56:14,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974767 2024-11-26T09:56:14,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974767 2024-11-26T09:56:14,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974767) roll requested 2024-11-26T09:56:14,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974774 2024-11-26T09:56:14,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974774 2024-11-26T09:56:14,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974774) roll requested 2024-11-26T09:56:14,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974780 2024-11-26T09:56:14,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974780 2024-11-26T09:56:14,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974780) roll requested 2024-11-26T09:56:14,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974787 2024-11-26T09:56:14,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974787 2024-11-26T09:56:14,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974787) roll requested 2024-11-26T09:56:14,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974793 2024-11-26T09:56:14,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974793 2024-11-26T09:56:14,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974793) roll requested 2024-11-26T09:56:14,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974799 2024-11-26T09:56:14,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974799 2024-11-26T09:56:14,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974799) roll requested 2024-11-26T09:56:14,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974805 2024-11-26T09:56:14,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974805 2024-11-26T09:56:14,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974805) roll requested 2024-11-26T09:56:14,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974811 2024-11-26T09:56:14,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974811 2024-11-26T09:56:14,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974811) roll requested 2024-11-26T09:56:14,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974817 2024-11-26T09:56:14,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974817 2024-11-26T09:56:14,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974817) roll requested 2024-11-26T09:56:14,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974823 2024-11-26T09:56:14,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974823 2024-11-26T09:56:14,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974823) roll requested 2024-11-26T09:56:14,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974830 2024-11-26T09:56:14,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974830 2024-11-26T09:56:14,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974830) roll requested 2024-11-26T09:56:14,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974836 2024-11-26T09:56:14,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974836 2024-11-26T09:56:14,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974836) roll requested 2024-11-26T09:56:14,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974842 2024-11-26T09:56:14,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974842 2024-11-26T09:56:14,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974842) roll requested 2024-11-26T09:56:14,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974848 2024-11-26T09:56:14,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974848 2024-11-26T09:56:14,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974848) roll requested 2024-11-26T09:56:14,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974855 2024-11-26T09:56:14,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974855 2024-11-26T09:56:14,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974855) roll requested 2024-11-26T09:56:14,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974861 2024-11-26T09:56:14,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974861 2024-11-26T09:56:14,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974861) roll requested 2024-11-26T09:56:14,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974867 2024-11-26T09:56:14,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974867 2024-11-26T09:56:14,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974867) roll requested 2024-11-26T09:56:14,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974873 2024-11-26T09:56:14,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974873 2024-11-26T09:56:14,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974873) roll requested 2024-11-26T09:56:14,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974879 2024-11-26T09:56:14,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974879 2024-11-26T09:56:14,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974879) roll requested 2024-11-26T09:56:14,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974885 2024-11-26T09:56:14,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974885 2024-11-26T09:56:14,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974885) roll requested 2024-11-26T09:56:14,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974892 2024-11-26T09:56:14,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974892 2024-11-26T09:56:14,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974892) roll requested 2024-11-26T09:56:14,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974898 2024-11-26T09:56:14,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974898 2024-11-26T09:56:14,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974898) roll requested 2024-11-26T09:56:14,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974904 2024-11-26T09:56:14,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974904 2024-11-26T09:56:14,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974904) roll requested 2024-11-26T09:56:14,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974910 2024-11-26T09:56:14,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974910 2024-11-26T09:56:14,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974910) roll requested 2024-11-26T09:56:14,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974917 2024-11-26T09:56:14,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974917 2024-11-26T09:56:14,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974917) roll requested 2024-11-26T09:56:14,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974923 2024-11-26T09:56:14,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974923 2024-11-26T09:56:14,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974923) roll requested 2024-11-26T09:56:14,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974929 2024-11-26T09:56:14,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974929 2024-11-26T09:56:14,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974929) roll requested 2024-11-26T09:56:14,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974935 2024-11-26T09:56:14,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974935 2024-11-26T09:56:14,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974935) roll requested 2024-11-26T09:56:14,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974941 2024-11-26T09:56:14,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974941 2024-11-26T09:56:14,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974941) roll requested 2024-11-26T09:56:14,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974948 2024-11-26T09:56:14,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974948 2024-11-26T09:56:14,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974948) roll requested 2024-11-26T09:56:14,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974954 2024-11-26T09:56:14,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974954 2024-11-26T09:56:14,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974954) roll requested 2024-11-26T09:56:14,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974961 2024-11-26T09:56:14,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974961 2024-11-26T09:56:14,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974961) roll requested 2024-11-26T09:56:14,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974967 2024-11-26T09:56:14,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974967 2024-11-26T09:56:14,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974967) roll requested 2024-11-26T09:56:14,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974973 2024-11-26T09:56:14,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974973 2024-11-26T09:56:14,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974973) roll requested 2024-11-26T09:56:14,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974979 2024-11-26T09:56:14,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974979 2024-11-26T09:56:14,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974979) roll requested 2024-11-26T09:56:14,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974986 2024-11-26T09:56:14,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974986 2024-11-26T09:56:14,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974986) roll requested 2024-11-26T09:56:14,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974992 2024-11-26T09:56:14,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:14,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974992 2024-11-26T09:56:14,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:14,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:14,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974992) roll requested 2024-11-26T09:56:14,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614974998 2024-11-26T09:56:15,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974998 2024-11-26T09:56:15,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614974998) roll requested 2024-11-26T09:56:15,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975004 2024-11-26T09:56:15,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614974998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975004 2024-11-26T09:56:15,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975004) roll requested 2024-11-26T09:56:15,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975011 2024-11-26T09:56:15,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975011 2024-11-26T09:56:15,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975011) roll requested 2024-11-26T09:56:15,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975017 2024-11-26T09:56:15,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975017 2024-11-26T09:56:15,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975017) roll requested 2024-11-26T09:56:15,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975023 2024-11-26T09:56:15,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975023 2024-11-26T09:56:15,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975023) roll requested 2024-11-26T09:56:15,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975029 2024-11-26T09:56:15,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975029 2024-11-26T09:56:15,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975029) roll requested 2024-11-26T09:56:15,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975035 2024-11-26T09:56:15,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975035 2024-11-26T09:56:15,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975035) roll requested 2024-11-26T09:56:15,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975043 2024-11-26T09:56:15,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975043 2024-11-26T09:56:15,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975043) roll requested 2024-11-26T09:56:15,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975052 2024-11-26T09:56:15,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975052 2024-11-26T09:56:15,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975052) roll requested 2024-11-26T09:56:15,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975062 2024-11-26T09:56:15,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975062 2024-11-26T09:56:15,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975062) roll requested 2024-11-26T09:56:15,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975070 2024-11-26T09:56:15,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975070 2024-11-26T09:56:15,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975070) roll requested 2024-11-26T09:56:15,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975077 2024-11-26T09:56:15,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975077 2024-11-26T09:56:15,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975077) roll requested 2024-11-26T09:56:15,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975085 2024-11-26T09:56:15,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975085 2024-11-26T09:56:15,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975085) roll requested 2024-11-26T09:56:15,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975093 2024-11-26T09:56:15,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975093 2024-11-26T09:56:15,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975093) roll requested 2024-11-26T09:56:15,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975103 2024-11-26T09:56:15,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975103 2024-11-26T09:56:15,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975103) roll requested 2024-11-26T09:56:15,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975121 2024-11-26T09:56:15,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975121 2024-11-26T09:56:15,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975121) roll requested 2024-11-26T09:56:15,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975127 2024-11-26T09:56:15,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975127 2024-11-26T09:56:15,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975127) roll requested 2024-11-26T09:56:15,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975133 2024-11-26T09:56:15,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975133 2024-11-26T09:56:15,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975133) roll requested 2024-11-26T09:56:15,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975140 2024-11-26T09:56:15,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975140 2024-11-26T09:56:15,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975140) roll requested 2024-11-26T09:56:15,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975146 2024-11-26T09:56:15,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975146 2024-11-26T09:56:15,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975146) roll requested 2024-11-26T09:56:15,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975153 2024-11-26T09:56:15,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975153 2024-11-26T09:56:15,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975153) roll requested 2024-11-26T09:56:15,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975159 2024-11-26T09:56:15,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975159 2024-11-26T09:56:15,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975159) roll requested 2024-11-26T09:56:15,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975165 2024-11-26T09:56:15,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975165 2024-11-26T09:56:15,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975165) roll requested 2024-11-26T09:56:15,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975171 2024-11-26T09:56:15,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975171 2024-11-26T09:56:15,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975171) roll requested 2024-11-26T09:56:15,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975177 2024-11-26T09:56:15,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975177 2024-11-26T09:56:15,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975177) roll requested 2024-11-26T09:56:15,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975184 2024-11-26T09:56:15,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975184 2024-11-26T09:56:15,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975184) roll requested 2024-11-26T09:56:15,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975190 2024-11-26T09:56:15,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975190 2024-11-26T09:56:15,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975190) roll requested 2024-11-26T09:56:15,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975196 2024-11-26T09:56:15,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975196 2024-11-26T09:56:15,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975196) roll requested 2024-11-26T09:56:15,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975202 2024-11-26T09:56:15,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975202 2024-11-26T09:56:15,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975202) roll requested 2024-11-26T09:56:15,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975208 2024-11-26T09:56:15,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975208 2024-11-26T09:56:15,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975208) roll requested 2024-11-26T09:56:15,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975215 2024-11-26T09:56:15,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975215 2024-11-26T09:56:15,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975215) roll requested 2024-11-26T09:56:15,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975221 2024-11-26T09:56:15,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975221 2024-11-26T09:56:15,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975221) roll requested 2024-11-26T09:56:15,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975227 2024-11-26T09:56:15,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975227 2024-11-26T09:56:15,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975227) roll requested 2024-11-26T09:56:15,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975233 2024-11-26T09:56:15,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975233 2024-11-26T09:56:15,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975233) roll requested 2024-11-26T09:56:15,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975239 2024-11-26T09:56:15,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975239 2024-11-26T09:56:15,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975239) roll requested 2024-11-26T09:56:15,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975246 2024-11-26T09:56:15,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975246 2024-11-26T09:56:15,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975246) roll requested 2024-11-26T09:56:15,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975253 2024-11-26T09:56:15,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975253 2024-11-26T09:56:15,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975253) roll requested 2024-11-26T09:56:15,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975259 2024-11-26T09:56:15,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975259 2024-11-26T09:56:15,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975259) roll requested 2024-11-26T09:56:15,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975266 2024-11-26T09:56:15,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975266 2024-11-26T09:56:15,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975266) roll requested 2024-11-26T09:56:15,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975273 2024-11-26T09:56:15,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975273 2024-11-26T09:56:15,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975273) roll requested 2024-11-26T09:56:15,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975279 2024-11-26T09:56:15,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975279 2024-11-26T09:56:15,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975279) roll requested 2024-11-26T09:56:15,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975285 2024-11-26T09:56:15,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975285 2024-11-26T09:56:15,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,291 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975285) roll requested 2024-11-26T09:56:15,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975291 2024-11-26T09:56:15,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975291 2024-11-26T09:56:15,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975291) roll requested 2024-11-26T09:56:15,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975298 2024-11-26T09:56:15,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975291 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975298 2024-11-26T09:56:15,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975298) roll requested 2024-11-26T09:56:15,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975304 2024-11-26T09:56:15,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975304 2024-11-26T09:56:15,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975304) roll requested 2024-11-26T09:56:15,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975310 2024-11-26T09:56:15,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975310 2024-11-26T09:56:15,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975310) roll requested 2024-11-26T09:56:15,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975317 2024-11-26T09:56:15,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975317 2024-11-26T09:56:15,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975317) roll requested 2024-11-26T09:56:15,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975323 2024-11-26T09:56:15,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975323 2024-11-26T09:56:15,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975323) roll requested 2024-11-26T09:56:15,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975329 2024-11-26T09:56:15,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975329 2024-11-26T09:56:15,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975329) roll requested 2024-11-26T09:56:15,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975337 2024-11-26T09:56:15,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975337 2024-11-26T09:56:15,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975337) roll requested 2024-11-26T09:56:15,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975343 2024-11-26T09:56:15,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975343 2024-11-26T09:56:15,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975343) roll requested 2024-11-26T09:56:15,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975350 2024-11-26T09:56:15,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975350 2024-11-26T09:56:15,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975350) roll requested 2024-11-26T09:56:15,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975356 2024-11-26T09:56:15,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975356 2024-11-26T09:56:15,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975356) roll requested 2024-11-26T09:56:15,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975362 2024-11-26T09:56:15,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975362 2024-11-26T09:56:15,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975362) roll requested 2024-11-26T09:56:15,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975368 2024-11-26T09:56:15,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975368 2024-11-26T09:56:15,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975368) roll requested 2024-11-26T09:56:15,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975374 2024-11-26T09:56:15,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975374 2024-11-26T09:56:15,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975374) roll requested 2024-11-26T09:56:15,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975381 2024-11-26T09:56:15,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975381 2024-11-26T09:56:15,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975381) roll requested 2024-11-26T09:56:15,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975388 2024-11-26T09:56:15,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975388 2024-11-26T09:56:15,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975388) roll requested 2024-11-26T09:56:15,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975394 2024-11-26T09:56:15,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975394 2024-11-26T09:56:15,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975394) roll requested 2024-11-26T09:56:15,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975402 2024-11-26T09:56:15,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975402 2024-11-26T09:56:15,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975402) roll requested 2024-11-26T09:56:15,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975408 2024-11-26T09:56:15,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975408 2024-11-26T09:56:15,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975408) roll requested 2024-11-26T09:56:15,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975415 2024-11-26T09:56:15,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975415 2024-11-26T09:56:15,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975415) roll requested 2024-11-26T09:56:15,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975422 2024-11-26T09:56:15,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975422 2024-11-26T09:56:15,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975422) roll requested 2024-11-26T09:56:15,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975431 2024-11-26T09:56:15,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975431 2024-11-26T09:56:15,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975431) roll requested 2024-11-26T09:56:15,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975443 2024-11-26T09:56:15,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975443 2024-11-26T09:56:15,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975443) roll requested 2024-11-26T09:56:15,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975450 2024-11-26T09:56:15,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975450 2024-11-26T09:56:15,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975450) roll requested 2024-11-26T09:56:15,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975457 2024-11-26T09:56:15,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975457 2024-11-26T09:56:15,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975457) roll requested 2024-11-26T09:56:15,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975463 2024-11-26T09:56:15,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975463 2024-11-26T09:56:15,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975463) roll requested 2024-11-26T09:56:15,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975469 2024-11-26T09:56:15,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975469 2024-11-26T09:56:15,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975469) roll requested 2024-11-26T09:56:15,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975476 2024-11-26T09:56:15,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975476 2024-11-26T09:56:15,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975476) roll requested 2024-11-26T09:56:15,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975482 2024-11-26T09:56:15,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975482 2024-11-26T09:56:15,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975482) roll requested 2024-11-26T09:56:15,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975489 2024-11-26T09:56:15,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975489 2024-11-26T09:56:15,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975489) roll requested 2024-11-26T09:56:15,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975495 2024-11-26T09:56:15,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975495 2024-11-26T09:56:15,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975495) roll requested 2024-11-26T09:56:15,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975501 2024-11-26T09:56:15,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975501 2024-11-26T09:56:15,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975501) roll requested 2024-11-26T09:56:15,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975508 2024-11-26T09:56:15,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975508 2024-11-26T09:56:15,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975508) roll requested 2024-11-26T09:56:15,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975515 2024-11-26T09:56:15,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975515 2024-11-26T09:56:15,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975515) roll requested 2024-11-26T09:56:15,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975521 2024-11-26T09:56:15,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975521 2024-11-26T09:56:15,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975521) roll requested 2024-11-26T09:56:15,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975527 2024-11-26T09:56:15,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975527 2024-11-26T09:56:15,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975527) roll requested 2024-11-26T09:56:15,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975534 2024-11-26T09:56:15,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975534 2024-11-26T09:56:15,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975534) roll requested 2024-11-26T09:56:15,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975541 2024-11-26T09:56:15,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975541 2024-11-26T09:56:15,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975541) roll requested 2024-11-26T09:56:15,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975547 2024-11-26T09:56:15,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975547 2024-11-26T09:56:15,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975547) roll requested 2024-11-26T09:56:15,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975553 2024-11-26T09:56:15,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975553 2024-11-26T09:56:15,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975553) roll requested 2024-11-26T09:56:15,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975560 2024-11-26T09:56:15,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975560 2024-11-26T09:56:15,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975560) roll requested 2024-11-26T09:56:15,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975566 2024-11-26T09:56:15,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975566 2024-11-26T09:56:15,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975566) roll requested 2024-11-26T09:56:15,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975572 2024-11-26T09:56:15,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975572 2024-11-26T09:56:15,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975572) roll requested 2024-11-26T09:56:15,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975579 2024-11-26T09:56:15,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975579 2024-11-26T09:56:15,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975579) roll requested 2024-11-26T09:56:15,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975585 2024-11-26T09:56:15,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975585 2024-11-26T09:56:15,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975585) roll requested 2024-11-26T09:56:15,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975591 2024-11-26T09:56:15,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975591 2024-11-26T09:56:15,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975591) roll requested 2024-11-26T09:56:15,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975598 2024-11-26T09:56:15,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975598 2024-11-26T09:56:15,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975598) roll requested 2024-11-26T09:56:15,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975604 2024-11-26T09:56:15,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975604 2024-11-26T09:56:15,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975604) roll requested 2024-11-26T09:56:15,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975610 2024-11-26T09:56:15,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975610 2024-11-26T09:56:15,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975610) roll requested 2024-11-26T09:56:15,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975617 2024-11-26T09:56:15,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975617 2024-11-26T09:56:15,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975617) roll requested 2024-11-26T09:56:15,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975623 2024-11-26T09:56:15,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975623 2024-11-26T09:56:15,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975623) roll requested 2024-11-26T09:56:15,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975629 2024-11-26T09:56:15,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975629 2024-11-26T09:56:15,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975629) roll requested 2024-11-26T09:56:15,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975636 2024-11-26T09:56:15,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975636 2024-11-26T09:56:15,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975636) roll requested 2024-11-26T09:56:15,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975642 2024-11-26T09:56:15,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975642 2024-11-26T09:56:15,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975642) roll requested 2024-11-26T09:56:15,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975649 2024-11-26T09:56:15,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975649 2024-11-26T09:56:15,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975649) roll requested 2024-11-26T09:56:15,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975655 2024-11-26T09:56:15,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975655 2024-11-26T09:56:15,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975655) roll requested 2024-11-26T09:56:15,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975661 2024-11-26T09:56:15,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975661 2024-11-26T09:56:15,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975661) roll requested 2024-11-26T09:56:15,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975667 2024-11-26T09:56:15,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975667 2024-11-26T09:56:15,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975667) roll requested 2024-11-26T09:56:15,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975674 2024-11-26T09:56:15,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975674 2024-11-26T09:56:15,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975674) roll requested 2024-11-26T09:56:15,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975681 2024-11-26T09:56:15,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975681 2024-11-26T09:56:15,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975681) roll requested 2024-11-26T09:56:15,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975687 2024-11-26T09:56:15,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975687 2024-11-26T09:56:15,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975687) roll requested 2024-11-26T09:56:15,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975693 2024-11-26T09:56:15,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975693 2024-11-26T09:56:15,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975693) roll requested 2024-11-26T09:56:15,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975700 2024-11-26T09:56:15,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975700 2024-11-26T09:56:15,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975700) roll requested 2024-11-26T09:56:15,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975706 2024-11-26T09:56:15,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975706 2024-11-26T09:56:15,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975706) roll requested 2024-11-26T09:56:15,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975712 2024-11-26T09:56:15,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975712 2024-11-26T09:56:15,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975712) roll requested 2024-11-26T09:56:15,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975718 2024-11-26T09:56:15,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975718 2024-11-26T09:56:15,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975718) roll requested 2024-11-26T09:56:15,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975725 2024-11-26T09:56:15,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975725 2024-11-26T09:56:15,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975725) roll requested 2024-11-26T09:56:15,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975732 2024-11-26T09:56:15,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975732 2024-11-26T09:56:15,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975732) roll requested 2024-11-26T09:56:15,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975747 2024-11-26T09:56:15,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975747 2024-11-26T09:56:15,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975747) roll requested 2024-11-26T09:56:15,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975757 2024-11-26T09:56:15,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975757 2024-11-26T09:56:15,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975757) roll requested 2024-11-26T09:56:15,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975765 2024-11-26T09:56:15,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975765 2024-11-26T09:56:15,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975765) roll requested 2024-11-26T09:56:15,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975776 2024-11-26T09:56:15,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975776 2024-11-26T09:56:15,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975776) roll requested 2024-11-26T09:56:15,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975784 2024-11-26T09:56:15,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975784 2024-11-26T09:56:15,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975784) roll requested 2024-11-26T09:56:15,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975800 2024-11-26T09:56:15,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975800 2024-11-26T09:56:15,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975800) roll requested 2024-11-26T09:56:15,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975811 2024-11-26T09:56:15,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975811 2024-11-26T09:56:15,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975811) roll requested 2024-11-26T09:56:15,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975820 2024-11-26T09:56:15,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975820 2024-11-26T09:56:15,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975820) roll requested 2024-11-26T09:56:15,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975826 2024-11-26T09:56:15,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975826 2024-11-26T09:56:15,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975826) roll requested 2024-11-26T09:56:15,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975833 2024-11-26T09:56:15,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975833 2024-11-26T09:56:15,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975833) roll requested 2024-11-26T09:56:15,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975840 2024-11-26T09:56:15,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975840 2024-11-26T09:56:15,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975840) roll requested 2024-11-26T09:56:15,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975846 2024-11-26T09:56:15,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975846 2024-11-26T09:56:15,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975846) roll requested 2024-11-26T09:56:15,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975852 2024-11-26T09:56:15,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975852 2024-11-26T09:56:15,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975852) roll requested 2024-11-26T09:56:15,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975858 2024-11-26T09:56:15,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975858 2024-11-26T09:56:15,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975858) roll requested 2024-11-26T09:56:15,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975865 2024-11-26T09:56:15,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975865 2024-11-26T09:56:15,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975865) roll requested 2024-11-26T09:56:15,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975871 2024-11-26T09:56:15,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975871 2024-11-26T09:56:15,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975871) roll requested 2024-11-26T09:56:15,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975877 2024-11-26T09:56:15,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975877 2024-11-26T09:56:15,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975877) roll requested 2024-11-26T09:56:15,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975883 2024-11-26T09:56:15,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975883 2024-11-26T09:56:15,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975883) roll requested 2024-11-26T09:56:15,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975891 2024-11-26T09:56:15,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975891 2024-11-26T09:56:15,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975891) roll requested 2024-11-26T09:56:15,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975898 2024-11-26T09:56:15,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975898 2024-11-26T09:56:15,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975898) roll requested 2024-11-26T09:56:15,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975905 2024-11-26T09:56:15,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975905 2024-11-26T09:56:15,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975905) roll requested 2024-11-26T09:56:15,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975911 2024-11-26T09:56:15,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975911 2024-11-26T09:56:15,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975911) roll requested 2024-11-26T09:56:15,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975918 2024-11-26T09:56:15,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975918 2024-11-26T09:56:15,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975918) roll requested 2024-11-26T09:56:15,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975924 2024-11-26T09:56:15,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975924 2024-11-26T09:56:15,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975924) roll requested 2024-11-26T09:56:15,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975930 2024-11-26T09:56:15,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975930 2024-11-26T09:56:15,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975930) roll requested 2024-11-26T09:56:15,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975937 2024-11-26T09:56:15,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975937 2024-11-26T09:56:15,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975937) roll requested 2024-11-26T09:56:15,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975944 2024-11-26T09:56:15,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975944 2024-11-26T09:56:15,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975944) roll requested 2024-11-26T09:56:15,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975952 2024-11-26T09:56:15,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975952 2024-11-26T09:56:15,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975952) roll requested 2024-11-26T09:56:15,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975958 2024-11-26T09:56:15,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975958 2024-11-26T09:56:15,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975958) roll requested 2024-11-26T09:56:15,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975965 2024-11-26T09:56:15,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975965 2024-11-26T09:56:15,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975965) roll requested 2024-11-26T09:56:15,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975971 2024-11-26T09:56:15,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975971 2024-11-26T09:56:15,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975971) roll requested 2024-11-26T09:56:15,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975978 2024-11-26T09:56:15,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975978 2024-11-26T09:56:15,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975978) roll requested 2024-11-26T09:56:15,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975984 2024-11-26T09:56:15,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975984 2024-11-26T09:56:15,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975984) roll requested 2024-11-26T09:56:15,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975991 2024-11-26T09:56:15,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:15,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975991 2024-11-26T09:56:15,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:15,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:15,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975991) roll requested 2024-11-26T09:56:15,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614975997 2024-11-26T09:56:16,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975997 2024-11-26T09:56:16,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614975997) roll requested 2024-11-26T09:56:16,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976004 2024-11-26T09:56:16,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614975997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976004 2024-11-26T09:56:16,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976004) roll requested 2024-11-26T09:56:16,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976011 2024-11-26T09:56:16,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976011 2024-11-26T09:56:16,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976011) roll requested 2024-11-26T09:56:16,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976017 2024-11-26T09:56:16,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976017 2024-11-26T09:56:16,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976017) roll requested 2024-11-26T09:56:16,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976023 2024-11-26T09:56:16,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976023 2024-11-26T09:56:16,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976023) roll requested 2024-11-26T09:56:16,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976029 2024-11-26T09:56:16,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976029 2024-11-26T09:56:16,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976029) roll requested 2024-11-26T09:56:16,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976040 2024-11-26T09:56:16,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976040 2024-11-26T09:56:16,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976040) roll requested 2024-11-26T09:56:16,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976048 2024-11-26T09:56:16,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976048 2024-11-26T09:56:16,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976048) roll requested 2024-11-26T09:56:16,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976056 2024-11-26T09:56:16,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976056 2024-11-26T09:56:16,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976056) roll requested 2024-11-26T09:56:16,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976064 2024-11-26T09:56:16,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976064 2024-11-26T09:56:16,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976064) roll requested 2024-11-26T09:56:16,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976076 2024-11-26T09:56:16,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976076 2024-11-26T09:56:16,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976076) roll requested 2024-11-26T09:56:16,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976082 2024-11-26T09:56:16,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976082 2024-11-26T09:56:16,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976082) roll requested 2024-11-26T09:56:16,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976090 2024-11-26T09:56:16,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976090 2024-11-26T09:56:16,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976090) roll requested 2024-11-26T09:56:16,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976096 2024-11-26T09:56:16,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976096 2024-11-26T09:56:16,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976096) roll requested 2024-11-26T09:56:16,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976103 2024-11-26T09:56:16,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976103 2024-11-26T09:56:16,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976103) roll requested 2024-11-26T09:56:16,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976109 2024-11-26T09:56:16,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976109 2024-11-26T09:56:16,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976109) roll requested 2024-11-26T09:56:16,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976116 2024-11-26T09:56:16,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976116 2024-11-26T09:56:16,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976116) roll requested 2024-11-26T09:56:16,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976123 2024-11-26T09:56:16,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976123 2024-11-26T09:56:16,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976123) roll requested 2024-11-26T09:56:16,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976131 2024-11-26T09:56:16,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976131 2024-11-26T09:56:16,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976131) roll requested 2024-11-26T09:56:16,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976142 2024-11-26T09:56:16,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976142 2024-11-26T09:56:16,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976142) roll requested 2024-11-26T09:56:16,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976150 2024-11-26T09:56:16,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976150 2024-11-26T09:56:16,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976150) roll requested 2024-11-26T09:56:16,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976160 2024-11-26T09:56:16,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976160 2024-11-26T09:56:16,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976160) roll requested 2024-11-26T09:56:16,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976169 2024-11-26T09:56:16,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976169 2024-11-26T09:56:16,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976169) roll requested 2024-11-26T09:56:16,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976177 2024-11-26T09:56:16,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976177 2024-11-26T09:56:16,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976177) roll requested 2024-11-26T09:56:16,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976183 2024-11-26T09:56:16,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976183 2024-11-26T09:56:16,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976183) roll requested 2024-11-26T09:56:16,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976190 2024-11-26T09:56:16,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976190 2024-11-26T09:56:16,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976190) roll requested 2024-11-26T09:56:16,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976196 2024-11-26T09:56:16,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976196 2024-11-26T09:56:16,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976196) roll requested 2024-11-26T09:56:16,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976202 2024-11-26T09:56:16,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976202 2024-11-26T09:56:16,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976202) roll requested 2024-11-26T09:56:16,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976209 2024-11-26T09:56:16,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976209 2024-11-26T09:56:16,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976209) roll requested 2024-11-26T09:56:16,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976215 2024-11-26T09:56:16,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976215 2024-11-26T09:56:16,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976215) roll requested 2024-11-26T09:56:16,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976227 2024-11-26T09:56:16,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976227 2024-11-26T09:56:16,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976227) roll requested 2024-11-26T09:56:16,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976239 2024-11-26T09:56:16,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976239 2024-11-26T09:56:16,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976239) roll requested 2024-11-26T09:56:16,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976248 2024-11-26T09:56:16,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976248 2024-11-26T09:56:16,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976248) roll requested 2024-11-26T09:56:16,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976254 2024-11-26T09:56:16,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976254 2024-11-26T09:56:16,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976254) roll requested 2024-11-26T09:56:16,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976261 2024-11-26T09:56:16,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976261 2024-11-26T09:56:16,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976261) roll requested 2024-11-26T09:56:16,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976267 2024-11-26T09:56:16,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976267 2024-11-26T09:56:16,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976267) roll requested 2024-11-26T09:56:16,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976273 2024-11-26T09:56:16,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976273 2024-11-26T09:56:16,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976273) roll requested 2024-11-26T09:56:16,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976279 2024-11-26T09:56:16,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976279 2024-11-26T09:56:16,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976279) roll requested 2024-11-26T09:56:16,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976286 2024-11-26T09:56:16,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976286 2024-11-26T09:56:16,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976286) roll requested 2024-11-26T09:56:16,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976292 2024-11-26T09:56:16,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976292 2024-11-26T09:56:16,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976292) roll requested 2024-11-26T09:56:16,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976298 2024-11-26T09:56:16,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976298 2024-11-26T09:56:16,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976298) roll requested 2024-11-26T09:56:16,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976304 2024-11-26T09:56:16,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976304 2024-11-26T09:56:16,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976304) roll requested 2024-11-26T09:56:16,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976311 2024-11-26T09:56:16,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976311 2024-11-26T09:56:16,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976311) roll requested 2024-11-26T09:56:16,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976317 2024-11-26T09:56:16,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976317 2024-11-26T09:56:16,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976317) roll requested 2024-11-26T09:56:16,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976323 2024-11-26T09:56:16,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976323 2024-11-26T09:56:16,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976323) roll requested 2024-11-26T09:56:16,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976331 2024-11-26T09:56:16,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976331 2024-11-26T09:56:16,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976331) roll requested 2024-11-26T09:56:16,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976340 2024-11-26T09:56:16,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976340 2024-11-26T09:56:16,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976340) roll requested 2024-11-26T09:56:16,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976346 2024-11-26T09:56:16,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976346 2024-11-26T09:56:16,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976346) roll requested 2024-11-26T09:56:16,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976353 2024-11-26T09:56:16,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976353 2024-11-26T09:56:16,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976353) roll requested 2024-11-26T09:56:16,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976359 2024-11-26T09:56:16,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976359 2024-11-26T09:56:16,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976359) roll requested 2024-11-26T09:56:16,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976366 2024-11-26T09:56:16,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976366 2024-11-26T09:56:16,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976366) roll requested 2024-11-26T09:56:16,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976372 2024-11-26T09:56:16,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976372 2024-11-26T09:56:16,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976372) roll requested 2024-11-26T09:56:16,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976378 2024-11-26T09:56:16,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976378 2024-11-26T09:56:16,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976378) roll requested 2024-11-26T09:56:16,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976390 2024-11-26T09:56:16,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976390 2024-11-26T09:56:16,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976390) roll requested 2024-11-26T09:56:16,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976397 2024-11-26T09:56:16,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976397 2024-11-26T09:56:16,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976397) roll requested 2024-11-26T09:56:16,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976404 2024-11-26T09:56:16,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976404 2024-11-26T09:56:16,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976404) roll requested 2024-11-26T09:56:16,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976410 2024-11-26T09:56:16,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976410 2024-11-26T09:56:16,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976410) roll requested 2024-11-26T09:56:16,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976417 2024-11-26T09:56:16,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976417 2024-11-26T09:56:16,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976417) roll requested 2024-11-26T09:56:16,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976424 2024-11-26T09:56:16,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976424 2024-11-26T09:56:16,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976424) roll requested 2024-11-26T09:56:16,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976431 2024-11-26T09:56:16,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976431 2024-11-26T09:56:16,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976431) roll requested 2024-11-26T09:56:16,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976444 2024-11-26T09:56:16,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976444 2024-11-26T09:56:16,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976444) roll requested 2024-11-26T09:56:16,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976457 2024-11-26T09:56:16,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976457 2024-11-26T09:56:16,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976457) roll requested 2024-11-26T09:56:16,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976468 2024-11-26T09:56:16,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976468 2024-11-26T09:56:16,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976468) roll requested 2024-11-26T09:56:16,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976478 2024-11-26T09:56:16,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976478 2024-11-26T09:56:16,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976478) roll requested 2024-11-26T09:56:16,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976486 2024-11-26T09:56:16,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976486 2024-11-26T09:56:16,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976486) roll requested 2024-11-26T09:56:16,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976497 2024-11-26T09:56:16,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976497 2024-11-26T09:56:16,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976497) roll requested 2024-11-26T09:56:16,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976513 2024-11-26T09:56:16,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976513 2024-11-26T09:56:16,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,529 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976513) roll requested 2024-11-26T09:56:16,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976530 2024-11-26T09:56:16,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976530 2024-11-26T09:56:16,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976530) roll requested 2024-11-26T09:56:16,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976540 2024-11-26T09:56:16,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976540 2024-11-26T09:56:16,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976540) roll requested 2024-11-26T09:56:16,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976553 2024-11-26T09:56:16,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976553 2024-11-26T09:56:16,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976553) roll requested 2024-11-26T09:56:16,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976580 2024-11-26T09:56:16,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976580 2024-11-26T09:56:16,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976580) roll requested 2024-11-26T09:56:16,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976594 2024-11-26T09:56:16,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976594 2024-11-26T09:56:16,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976594) roll requested 2024-11-26T09:56:16,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976625 2024-11-26T09:56:16,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976625 2024-11-26T09:56:16,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976625) roll requested 2024-11-26T09:56:16,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976640 2024-11-26T09:56:16,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976640 2024-11-26T09:56:16,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976640) roll requested 2024-11-26T09:56:16,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976653 2024-11-26T09:56:16,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976653 2024-11-26T09:56:16,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976653) roll requested 2024-11-26T09:56:16,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976663 2024-11-26T09:56:16,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976663 2024-11-26T09:56:16,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976663) roll requested 2024-11-26T09:56:16,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976676 2024-11-26T09:56:16,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976676 2024-11-26T09:56:16,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976676) roll requested 2024-11-26T09:56:16,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976686 2024-11-26T09:56:16,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976686 2024-11-26T09:56:16,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976686) roll requested 2024-11-26T09:56:16,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976697 2024-11-26T09:56:16,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976697 2024-11-26T09:56:16,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976697) roll requested 2024-11-26T09:56:16,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976715 2024-11-26T09:56:16,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976715 2024-11-26T09:56:16,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976715) roll requested 2024-11-26T09:56:16,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976724 2024-11-26T09:56:16,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976724 2024-11-26T09:56:16,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976724) roll requested 2024-11-26T09:56:16,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976735 2024-11-26T09:56:16,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976735 2024-11-26T09:56:16,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976735) roll requested 2024-11-26T09:56:16,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976748 2024-11-26T09:56:16,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976748 2024-11-26T09:56:16,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976748) roll requested 2024-11-26T09:56:16,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976764 2024-11-26T09:56:16,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976764 2024-11-26T09:56:16,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976764) roll requested 2024-11-26T09:56:16,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976787 2024-11-26T09:56:16,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976787 2024-11-26T09:56:16,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976787) roll requested 2024-11-26T09:56:16,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976799 2024-11-26T09:56:16,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976799 2024-11-26T09:56:16,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976799) roll requested 2024-11-26T09:56:16,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976809 2024-11-26T09:56:16,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976809 2024-11-26T09:56:16,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976809) roll requested 2024-11-26T09:56:16,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976820 2024-11-26T09:56:16,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976820 2024-11-26T09:56:16,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976820) roll requested 2024-11-26T09:56:16,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976832 2024-11-26T09:56:16,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976832 2024-11-26T09:56:16,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976832) roll requested 2024-11-26T09:56:16,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976841 2024-11-26T09:56:16,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976841 2024-11-26T09:56:16,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976841) roll requested 2024-11-26T09:56:16,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976848 2024-11-26T09:56:16,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976848 2024-11-26T09:56:16,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976848) roll requested 2024-11-26T09:56:16,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976856 2024-11-26T09:56:16,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976856 2024-11-26T09:56:16,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976856) roll requested 2024-11-26T09:56:16,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976865 2024-11-26T09:56:16,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976865 2024-11-26T09:56:16,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976865) roll requested 2024-11-26T09:56:16,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976873 2024-11-26T09:56:16,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976873 2024-11-26T09:56:16,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976873) roll requested 2024-11-26T09:56:16,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976885 2024-11-26T09:56:16,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976885 2024-11-26T09:56:16,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976885) roll requested 2024-11-26T09:56:16,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976900 2024-11-26T09:56:16,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976900 2024-11-26T09:56:16,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976900) roll requested 2024-11-26T09:56:16,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976916 2024-11-26T09:56:16,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976916 2024-11-26T09:56:16,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976916) roll requested 2024-11-26T09:56:16,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976928 2024-11-26T09:56:16,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976928 2024-11-26T09:56:16,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976928) roll requested 2024-11-26T09:56:16,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976942 2024-11-26T09:56:16,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976942 2024-11-26T09:56:16,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976942) roll requested 2024-11-26T09:56:16,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976961 2024-11-26T09:56:16,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976961 2024-11-26T09:56:16,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976961) roll requested 2024-11-26T09:56:16,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976972 2024-11-26T09:56:16,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976972 2024-11-26T09:56:16,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976972) roll requested 2024-11-26T09:56:16,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976980 2024-11-26T09:56:16,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976980 2024-11-26T09:56:16,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:16,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:16,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976980) roll requested 2024-11-26T09:56:16,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614976987 2024-11-26T09:56:16,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:16,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976987 2024-11-26T09:56:17,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614976987) roll requested 2024-11-26T09:56:17,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977001 2024-11-26T09:56:17,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614976987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977001 2024-11-26T09:56:17,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977001) roll requested 2024-11-26T09:56:17,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977012 2024-11-26T09:56:17,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977012 2024-11-26T09:56:17,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977012) roll requested 2024-11-26T09:56:17,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977023 2024-11-26T09:56:17,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977023 2024-11-26T09:56:17,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977023) roll requested 2024-11-26T09:56:17,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977030 2024-11-26T09:56:17,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977030 2024-11-26T09:56:17,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977030) roll requested 2024-11-26T09:56:17,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977037 2024-11-26T09:56:17,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977037 2024-11-26T09:56:17,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977037) roll requested 2024-11-26T09:56:17,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977052 2024-11-26T09:56:17,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977052 2024-11-26T09:56:17,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977052) roll requested 2024-11-26T09:56:17,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977064 2024-11-26T09:56:17,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977064 2024-11-26T09:56:17,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977064) roll requested 2024-11-26T09:56:17,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977073 2024-11-26T09:56:17,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977073 2024-11-26T09:56:17,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977073) roll requested 2024-11-26T09:56:17,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977083 2024-11-26T09:56:17,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977083 2024-11-26T09:56:17,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977083) roll requested 2024-11-26T09:56:17,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977089 2024-11-26T09:56:17,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977089 2024-11-26T09:56:17,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977089) roll requested 2024-11-26T09:56:17,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977096 2024-11-26T09:56:17,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977096 2024-11-26T09:56:17,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977096) roll requested 2024-11-26T09:56:17,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977102 2024-11-26T09:56:17,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977102 2024-11-26T09:56:17,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977102) roll requested 2024-11-26T09:56:17,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977109 2024-11-26T09:56:17,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977109 2024-11-26T09:56:17,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977109) roll requested 2024-11-26T09:56:17,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977115 2024-11-26T09:56:17,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977115 2024-11-26T09:56:17,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977115) roll requested 2024-11-26T09:56:17,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977121 2024-11-26T09:56:17,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977121 2024-11-26T09:56:17,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977121) roll requested 2024-11-26T09:56:17,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977127 2024-11-26T09:56:17,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977127 2024-11-26T09:56:17,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977127) roll requested 2024-11-26T09:56:17,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977134 2024-11-26T09:56:17,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977134 2024-11-26T09:56:17,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977134) roll requested 2024-11-26T09:56:17,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977140 2024-11-26T09:56:17,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977140 2024-11-26T09:56:17,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977140) roll requested 2024-11-26T09:56:17,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977146 2024-11-26T09:56:17,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977146 2024-11-26T09:56:17,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977146) roll requested 2024-11-26T09:56:17,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977153 2024-11-26T09:56:17,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977153 2024-11-26T09:56:17,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977153) roll requested 2024-11-26T09:56:17,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977163 2024-11-26T09:56:17,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977163 2024-11-26T09:56:17,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977163) roll requested 2024-11-26T09:56:17,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977170 2024-11-26T09:56:17,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977170 2024-11-26T09:56:17,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977170) roll requested 2024-11-26T09:56:17,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977176 2024-11-26T09:56:17,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977176 2024-11-26T09:56:17,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977176) roll requested 2024-11-26T09:56:17,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977182 2024-11-26T09:56:17,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977182 2024-11-26T09:56:17,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977182) roll requested 2024-11-26T09:56:17,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977189 2024-11-26T09:56:17,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977189 2024-11-26T09:56:17,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977189) roll requested 2024-11-26T09:56:17,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977195 2024-11-26T09:56:17,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977195 2024-11-26T09:56:17,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977195) roll requested 2024-11-26T09:56:17,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977201 2024-11-26T09:56:17,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977201 2024-11-26T09:56:17,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977201) roll requested 2024-11-26T09:56:17,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977207 2024-11-26T09:56:17,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977207 2024-11-26T09:56:17,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977207) roll requested 2024-11-26T09:56:17,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977213 2024-11-26T09:56:17,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977213 2024-11-26T09:56:17,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977213) roll requested 2024-11-26T09:56:17,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977219 2024-11-26T09:56:17,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977219 2024-11-26T09:56:17,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977219) roll requested 2024-11-26T09:56:17,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977226 2024-11-26T09:56:17,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977226 2024-11-26T09:56:17,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977226) roll requested 2024-11-26T09:56:17,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977232 2024-11-26T09:56:17,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977232 2024-11-26T09:56:17,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977232) roll requested 2024-11-26T09:56:17,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977238 2024-11-26T09:56:17,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977238 2024-11-26T09:56:17,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977238) roll requested 2024-11-26T09:56:17,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977245 2024-11-26T09:56:17,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977245 2024-11-26T09:56:17,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977245) roll requested 2024-11-26T09:56:17,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977251 2024-11-26T09:56:17,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977251 2024-11-26T09:56:17,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977251) roll requested 2024-11-26T09:56:17,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977257 2024-11-26T09:56:17,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977257 2024-11-26T09:56:17,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977257) roll requested 2024-11-26T09:56:17,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977263 2024-11-26T09:56:17,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977263 2024-11-26T09:56:17,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977263) roll requested 2024-11-26T09:56:17,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977270 2024-11-26T09:56:17,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977270 2024-11-26T09:56:17,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977270) roll requested 2024-11-26T09:56:17,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977278 2024-11-26T09:56:17,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977278 2024-11-26T09:56:17,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977278) roll requested 2024-11-26T09:56:17,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977286 2024-11-26T09:56:17,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977286 2024-11-26T09:56:17,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977286) roll requested 2024-11-26T09:56:17,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977294 2024-11-26T09:56:17,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977294 2024-11-26T09:56:17,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977294) roll requested 2024-11-26T09:56:17,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977303 2024-11-26T09:56:17,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977303 2024-11-26T09:56:17,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977303) roll requested 2024-11-26T09:56:17,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977314 2024-11-26T09:56:17,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977314 2024-11-26T09:56:17,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977314) roll requested 2024-11-26T09:56:17,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977320 2024-11-26T09:56:17,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977320 2024-11-26T09:56:17,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977320) roll requested 2024-11-26T09:56:17,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977328 2024-11-26T09:56:17,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977328 2024-11-26T09:56:17,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977328) roll requested 2024-11-26T09:56:17,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977336 2024-11-26T09:56:17,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977336 2024-11-26T09:56:17,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977336) roll requested 2024-11-26T09:56:17,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977343 2024-11-26T09:56:17,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977343 2024-11-26T09:56:17,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977343) roll requested 2024-11-26T09:56:17,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977351 2024-11-26T09:56:17,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977351 2024-11-26T09:56:17,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977351) roll requested 2024-11-26T09:56:17,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977358 2024-11-26T09:56:17,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977358 2024-11-26T09:56:17,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977358) roll requested 2024-11-26T09:56:17,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977364 2024-11-26T09:56:17,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977364 2024-11-26T09:56:17,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977364) roll requested 2024-11-26T09:56:17,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977371 2024-11-26T09:56:17,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977371 2024-11-26T09:56:17,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977371) roll requested 2024-11-26T09:56:17,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977379 2024-11-26T09:56:17,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977379 2024-11-26T09:56:17,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977379) roll requested 2024-11-26T09:56:17,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977389 2024-11-26T09:56:17,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977389 2024-11-26T09:56:17,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977389) roll requested 2024-11-26T09:56:17,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977396 2024-11-26T09:56:17,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977396 2024-11-26T09:56:17,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977396) roll requested 2024-11-26T09:56:17,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977404 2024-11-26T09:56:17,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977404 2024-11-26T09:56:17,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977404) roll requested 2024-11-26T09:56:17,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977411 2024-11-26T09:56:17,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977411 2024-11-26T09:56:17,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977411) roll requested 2024-11-26T09:56:17,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977418 2024-11-26T09:56:17,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977418 2024-11-26T09:56:17,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977418) roll requested 2024-11-26T09:56:17,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977424 2024-11-26T09:56:17,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977424 2024-11-26T09:56:17,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977424) roll requested 2024-11-26T09:56:17,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977430 2024-11-26T09:56:17,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977430 2024-11-26T09:56:17,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977430) roll requested 2024-11-26T09:56:17,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977437 2024-11-26T09:56:17,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977437 2024-11-26T09:56:17,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977437) roll requested 2024-11-26T09:56:17,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977444 2024-11-26T09:56:17,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977444 2024-11-26T09:56:17,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977444) roll requested 2024-11-26T09:56:17,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977457 2024-11-26T09:56:17,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977457 2024-11-26T09:56:17,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977457) roll requested 2024-11-26T09:56:17,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977467 2024-11-26T09:56:17,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977467 2024-11-26T09:56:17,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977467) roll requested 2024-11-26T09:56:17,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977474 2024-11-26T09:56:17,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977474 2024-11-26T09:56:17,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977474) roll requested 2024-11-26T09:56:17,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977480 2024-11-26T09:56:17,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977480 2024-11-26T09:56:17,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977480) roll requested 2024-11-26T09:56:17,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977487 2024-11-26T09:56:17,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977487 2024-11-26T09:56:17,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977487) roll requested 2024-11-26T09:56:17,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977493 2024-11-26T09:56:17,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977493 2024-11-26T09:56:17,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977493) roll requested 2024-11-26T09:56:17,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977501 2024-11-26T09:56:17,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977501 2024-11-26T09:56:17,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977501) roll requested 2024-11-26T09:56:17,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977507 2024-11-26T09:56:17,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977507 2024-11-26T09:56:17,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977507) roll requested 2024-11-26T09:56:17,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977514 2024-11-26T09:56:17,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977514 2024-11-26T09:56:17,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977514) roll requested 2024-11-26T09:56:17,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977520 2024-11-26T09:56:17,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977520 2024-11-26T09:56:17,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977520) roll requested 2024-11-26T09:56:17,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977526 2024-11-26T09:56:17,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977526 2024-11-26T09:56:17,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977526) roll requested 2024-11-26T09:56:17,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977532 2024-11-26T09:56:17,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977532 2024-11-26T09:56:17,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977532) roll requested 2024-11-26T09:56:17,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977539 2024-11-26T09:56:17,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977539 2024-11-26T09:56:17,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977539) roll requested 2024-11-26T09:56:17,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977545 2024-11-26T09:56:17,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977545 2024-11-26T09:56:17,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977545) roll requested 2024-11-26T09:56:17,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977551 2024-11-26T09:56:17,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977551 2024-11-26T09:56:17,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977551) roll requested 2024-11-26T09:56:17,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977558 2024-11-26T09:56:17,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977558 2024-11-26T09:56:17,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977558) roll requested 2024-11-26T09:56:17,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977564 2024-11-26T09:56:17,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977564 2024-11-26T09:56:17,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977564) roll requested 2024-11-26T09:56:17,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977570 2024-11-26T09:56:17,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977570 2024-11-26T09:56:17,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977570) roll requested 2024-11-26T09:56:17,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977576 2024-11-26T09:56:17,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977576 2024-11-26T09:56:17,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977576) roll requested 2024-11-26T09:56:17,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977582 2024-11-26T09:56:17,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977582 2024-11-26T09:56:17,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977582) roll requested 2024-11-26T09:56:17,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977589 2024-11-26T09:56:17,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977589 2024-11-26T09:56:17,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977589) roll requested 2024-11-26T09:56:17,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977595 2024-11-26T09:56:17,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977595 2024-11-26T09:56:17,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977595) roll requested 2024-11-26T09:56:17,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977602 2024-11-26T09:56:17,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977602 2024-11-26T09:56:17,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977602) roll requested 2024-11-26T09:56:17,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977608 2024-11-26T09:56:17,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977608 2024-11-26T09:56:17,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977608) roll requested 2024-11-26T09:56:17,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977614 2024-11-26T09:56:17,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977614 2024-11-26T09:56:17,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977614) roll requested 2024-11-26T09:56:17,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977620 2024-11-26T09:56:17,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977620 2024-11-26T09:56:17,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977620) roll requested 2024-11-26T09:56:17,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977626 2024-11-26T09:56:17,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977626 2024-11-26T09:56:17,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977626) roll requested 2024-11-26T09:56:17,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977632 2024-11-26T09:56:17,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977632 2024-11-26T09:56:17,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977632) roll requested 2024-11-26T09:56:17,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977643 2024-11-26T09:56:17,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977643 2024-11-26T09:56:17,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977643) roll requested 2024-11-26T09:56:17,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977649 2024-11-26T09:56:17,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977649 2024-11-26T09:56:17,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977649) roll requested 2024-11-26T09:56:17,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977655 2024-11-26T09:56:17,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977655 2024-11-26T09:56:17,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977655) roll requested 2024-11-26T09:56:17,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977662 2024-11-26T09:56:17,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977662 2024-11-26T09:56:17,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977662) roll requested 2024-11-26T09:56:17,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977668 2024-11-26T09:56:17,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977668 2024-11-26T09:56:17,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977668) roll requested 2024-11-26T09:56:17,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977674 2024-11-26T09:56:17,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977674 2024-11-26T09:56:17,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977674) roll requested 2024-11-26T09:56:17,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977680 2024-11-26T09:56:17,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977680 2024-11-26T09:56:17,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977680) roll requested 2024-11-26T09:56:17,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977686 2024-11-26T09:56:17,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977686 2024-11-26T09:56:17,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977686) roll requested 2024-11-26T09:56:17,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977693 2024-11-26T09:56:17,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977693 2024-11-26T09:56:17,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977693) roll requested 2024-11-26T09:56:17,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977698 2024-11-26T09:56:17,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977698 2024-11-26T09:56:17,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977698) roll requested 2024-11-26T09:56:17,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977705 2024-11-26T09:56:17,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977705 2024-11-26T09:56:17,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977705) roll requested 2024-11-26T09:56:17,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977711 2024-11-26T09:56:17,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977711 2024-11-26T09:56:17,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,717 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977711) roll requested 2024-11-26T09:56:17,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977717 2024-11-26T09:56:17,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977717 2024-11-26T09:56:17,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977717) roll requested 2024-11-26T09:56:17,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977723 2024-11-26T09:56:17,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977723 2024-11-26T09:56:17,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977723) roll requested 2024-11-26T09:56:17,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977729 2024-11-26T09:56:17,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977729 2024-11-26T09:56:17,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977729) roll requested 2024-11-26T09:56:17,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977736 2024-11-26T09:56:17,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977736 2024-11-26T09:56:17,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977736) roll requested 2024-11-26T09:56:17,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977743 2024-11-26T09:56:17,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977743 2024-11-26T09:56:17,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977743) roll requested 2024-11-26T09:56:17,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977750 2024-11-26T09:56:17,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977750 2024-11-26T09:56:17,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977750) roll requested 2024-11-26T09:56:17,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977756 2024-11-26T09:56:17,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977756 2024-11-26T09:56:17,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977756) roll requested 2024-11-26T09:56:17,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977767 2024-11-26T09:56:17,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977767 2024-11-26T09:56:17,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977767) roll requested 2024-11-26T09:56:17,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977773 2024-11-26T09:56:17,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977773 2024-11-26T09:56:17,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977773) roll requested 2024-11-26T09:56:17,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977780 2024-11-26T09:56:17,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977780 2024-11-26T09:56:17,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977780) roll requested 2024-11-26T09:56:17,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977786 2024-11-26T09:56:17,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977786 2024-11-26T09:56:17,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977786) roll requested 2024-11-26T09:56:17,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977796 2024-11-26T09:56:17,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977796 2024-11-26T09:56:17,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977796) roll requested 2024-11-26T09:56:17,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977802 2024-11-26T09:56:17,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977802 2024-11-26T09:56:17,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977802) roll requested 2024-11-26T09:56:17,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977814 2024-11-26T09:56:17,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977814 2024-11-26T09:56:17,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977814) roll requested 2024-11-26T09:56:17,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977821 2024-11-26T09:56:17,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977821 2024-11-26T09:56:17,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977821) roll requested 2024-11-26T09:56:17,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977828 2024-11-26T09:56:17,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977828 2024-11-26T09:56:17,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977828) roll requested 2024-11-26T09:56:17,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977834 2024-11-26T09:56:17,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977834 2024-11-26T09:56:17,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977834) roll requested 2024-11-26T09:56:17,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977840 2024-11-26T09:56:17,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977840 2024-11-26T09:56:17,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977840) roll requested 2024-11-26T09:56:17,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977847 2024-11-26T09:56:17,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977847 2024-11-26T09:56:17,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977847) roll requested 2024-11-26T09:56:17,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977854 2024-11-26T09:56:17,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977854 2024-11-26T09:56:17,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977854) roll requested 2024-11-26T09:56:17,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977860 2024-11-26T09:56:17,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977860 2024-11-26T09:56:17,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977860) roll requested 2024-11-26T09:56:17,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977867 2024-11-26T09:56:17,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977867 2024-11-26T09:56:17,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977867) roll requested 2024-11-26T09:56:17,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977874 2024-11-26T09:56:17,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977874 2024-11-26T09:56:17,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977874) roll requested 2024-11-26T09:56:17,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977881 2024-11-26T09:56:17,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977881 2024-11-26T09:56:17,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977881) roll requested 2024-11-26T09:56:17,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977890 2024-11-26T09:56:17,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977890 2024-11-26T09:56:17,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977890) roll requested 2024-11-26T09:56:17,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977898 2024-11-26T09:56:17,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977898 2024-11-26T09:56:17,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977898) roll requested 2024-11-26T09:56:17,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977906 2024-11-26T09:56:17,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977906 2024-11-26T09:56:17,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977906) roll requested 2024-11-26T09:56:17,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977915 2024-11-26T09:56:17,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977915 2024-11-26T09:56:17,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977915) roll requested 2024-11-26T09:56:17,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977924 2024-11-26T09:56:17,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977924 2024-11-26T09:56:17,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977924) roll requested 2024-11-26T09:56:17,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977938 2024-11-26T09:56:17,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977938 2024-11-26T09:56:17,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977938) roll requested 2024-11-26T09:56:17,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977946 2024-11-26T09:56:17,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977946 2024-11-26T09:56:17,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977946) roll requested 2024-11-26T09:56:17,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977954 2024-11-26T09:56:17,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977954 2024-11-26T09:56:17,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977954) roll requested 2024-11-26T09:56:17,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977964 2024-11-26T09:56:17,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977964 2024-11-26T09:56:17,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977964) roll requested 2024-11-26T09:56:17,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977977 2024-11-26T09:56:17,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977977 2024-11-26T09:56:17,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977977) roll requested 2024-11-26T09:56:17,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977987 2024-11-26T09:56:17,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:17,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977987 2024-11-26T09:56:17,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:17,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:17,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977987) roll requested 2024-11-26T09:56:17,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614977996 2024-11-26T09:56:18,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977996 2024-11-26T09:56:18,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614977996) roll requested 2024-11-26T09:56:18,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978002 2024-11-26T09:56:18,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614977996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978002 2024-11-26T09:56:18,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978002) roll requested 2024-11-26T09:56:18,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978009 2024-11-26T09:56:18,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978009 2024-11-26T09:56:18,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978009) roll requested 2024-11-26T09:56:18,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978015 2024-11-26T09:56:18,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978015 2024-11-26T09:56:18,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978015) roll requested 2024-11-26T09:56:18,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978021 2024-11-26T09:56:18,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978021 2024-11-26T09:56:18,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978021) roll requested 2024-11-26T09:56:18,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978028 2024-11-26T09:56:18,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978028 2024-11-26T09:56:18,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978028) roll requested 2024-11-26T09:56:18,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978035 2024-11-26T09:56:18,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978035 2024-11-26T09:56:18,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978035) roll requested 2024-11-26T09:56:18,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978048 2024-11-26T09:56:18,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978048 2024-11-26T09:56:18,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978048) roll requested 2024-11-26T09:56:18,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978056 2024-11-26T09:56:18,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978056 2024-11-26T09:56:18,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978056) roll requested 2024-11-26T09:56:18,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978062 2024-11-26T09:56:18,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978062 2024-11-26T09:56:18,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978062) roll requested 2024-11-26T09:56:18,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978069 2024-11-26T09:56:18,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978069 2024-11-26T09:56:18,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978069) roll requested 2024-11-26T09:56:18,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978075 2024-11-26T09:56:18,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978075 2024-11-26T09:56:18,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978075) roll requested 2024-11-26T09:56:18,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978081 2024-11-26T09:56:18,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978081 2024-11-26T09:56:18,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978081) roll requested 2024-11-26T09:56:18,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978089 2024-11-26T09:56:18,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978089 2024-11-26T09:56:18,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978089) roll requested 2024-11-26T09:56:18,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978096 2024-11-26T09:56:18,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978096 2024-11-26T09:56:18,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978096) roll requested 2024-11-26T09:56:18,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978110 2024-11-26T09:56:18,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978110 2024-11-26T09:56:18,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978110) roll requested 2024-11-26T09:56:18,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978118 2024-11-26T09:56:18,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978118 2024-11-26T09:56:18,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978118) roll requested 2024-11-26T09:56:18,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978125 2024-11-26T09:56:18,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978125 2024-11-26T09:56:18,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978125) roll requested 2024-11-26T09:56:18,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978134 2024-11-26T09:56:18,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978134 2024-11-26T09:56:18,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978134) roll requested 2024-11-26T09:56:18,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978145 2024-11-26T09:56:18,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978145 2024-11-26T09:56:18,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978145) roll requested 2024-11-26T09:56:18,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978153 2024-11-26T09:56:18,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978153 2024-11-26T09:56:18,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978153) roll requested 2024-11-26T09:56:18,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978160 2024-11-26T09:56:18,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978160 2024-11-26T09:56:18,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978160) roll requested 2024-11-26T09:56:18,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978167 2024-11-26T09:56:18,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978167 2024-11-26T09:56:18,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978167) roll requested 2024-11-26T09:56:18,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978176 2024-11-26T09:56:18,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978176 2024-11-26T09:56:18,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978176) roll requested 2024-11-26T09:56:18,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978186 2024-11-26T09:56:18,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978186 2024-11-26T09:56:18,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978186) roll requested 2024-11-26T09:56:18,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978195 2024-11-26T09:56:18,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978195 2024-11-26T09:56:18,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978195) roll requested 2024-11-26T09:56:18,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978204 2024-11-26T09:56:18,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978204 2024-11-26T09:56:18,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978204) roll requested 2024-11-26T09:56:18,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978212 2024-11-26T09:56:18,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978212 2024-11-26T09:56:18,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978212) roll requested 2024-11-26T09:56:18,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978221 2024-11-26T09:56:18,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978221 2024-11-26T09:56:18,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978221) roll requested 2024-11-26T09:56:18,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978229 2024-11-26T09:56:18,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978229 2024-11-26T09:56:18,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978229) roll requested 2024-11-26T09:56:18,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978236 2024-11-26T09:56:18,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978236 2024-11-26T09:56:18,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978236) roll requested 2024-11-26T09:56:18,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978248 2024-11-26T09:56:18,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978248 2024-11-26T09:56:18,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978248) roll requested 2024-11-26T09:56:18,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978268 2024-11-26T09:56:18,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978268 2024-11-26T09:56:18,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978268) roll requested 2024-11-26T09:56:18,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978277 2024-11-26T09:56:18,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978277 2024-11-26T09:56:18,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978277) roll requested 2024-11-26T09:56:18,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978288 2024-11-26T09:56:18,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978288 2024-11-26T09:56:18,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978288) roll requested 2024-11-26T09:56:18,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978296 2024-11-26T09:56:18,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978296 2024-11-26T09:56:18,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978296) roll requested 2024-11-26T09:56:18,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978304 2024-11-26T09:56:18,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978304 2024-11-26T09:56:18,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978304) roll requested 2024-11-26T09:56:18,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978311 2024-11-26T09:56:18,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978311 2024-11-26T09:56:18,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978311) roll requested 2024-11-26T09:56:18,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978323 2024-11-26T09:56:18,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978323 2024-11-26T09:56:18,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978323) roll requested 2024-11-26T09:56:18,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978332 2024-11-26T09:56:18,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978332 2024-11-26T09:56:18,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978332) roll requested 2024-11-26T09:56:18,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978340 2024-11-26T09:56:18,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978340 2024-11-26T09:56:18,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978340) roll requested 2024-11-26T09:56:18,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978347 2024-11-26T09:56:18,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978347 2024-11-26T09:56:18,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978347) roll requested 2024-11-26T09:56:18,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978355 2024-11-26T09:56:18,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978355 2024-11-26T09:56:18,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978355) roll requested 2024-11-26T09:56:18,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978363 2024-11-26T09:56:18,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978363 2024-11-26T09:56:18,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978363) roll requested 2024-11-26T09:56:18,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978372 2024-11-26T09:56:18,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978372 2024-11-26T09:56:18,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978372) roll requested 2024-11-26T09:56:18,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978379 2024-11-26T09:56:18,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978379 2024-11-26T09:56:18,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978379) roll requested 2024-11-26T09:56:18,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978387 2024-11-26T09:56:18,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978387 2024-11-26T09:56:18,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978387) roll requested 2024-11-26T09:56:18,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978396 2024-11-26T09:56:18,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978396 2024-11-26T09:56:18,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978396) roll requested 2024-11-26T09:56:18,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978403 2024-11-26T09:56:18,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978403 2024-11-26T09:56:18,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978403) roll requested 2024-11-26T09:56:18,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978411 2024-11-26T09:56:18,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978411 2024-11-26T09:56:18,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978411) roll requested 2024-11-26T09:56:18,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978421 2024-11-26T09:56:18,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978421 2024-11-26T09:56:18,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978421) roll requested 2024-11-26T09:56:18,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978427 2024-11-26T09:56:18,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978427 2024-11-26T09:56:18,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978427) roll requested 2024-11-26T09:56:18,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978437 2024-11-26T09:56:18,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978437 2024-11-26T09:56:18,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978437) roll requested 2024-11-26T09:56:18,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978448 2024-11-26T09:56:18,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978448 2024-11-26T09:56:18,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978448) roll requested 2024-11-26T09:56:18,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978457 2024-11-26T09:56:18,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978457 2024-11-26T09:56:18,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978457) roll requested 2024-11-26T09:56:18,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978467 2024-11-26T09:56:18,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978467 2024-11-26T09:56:18,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978467) roll requested 2024-11-26T09:56:18,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978483 2024-11-26T09:56:18,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978483 2024-11-26T09:56:18,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978483) roll requested 2024-11-26T09:56:18,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978491 2024-11-26T09:56:18,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978491 2024-11-26T09:56:18,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978491) roll requested 2024-11-26T09:56:18,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978502 2024-11-26T09:56:18,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978502 2024-11-26T09:56:18,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978502) roll requested 2024-11-26T09:56:18,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978515 2024-11-26T09:56:18,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978515 2024-11-26T09:56:18,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978515) roll requested 2024-11-26T09:56:18,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978528 2024-11-26T09:56:18,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978528 2024-11-26T09:56:18,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978528) roll requested 2024-11-26T09:56:18,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978545 2024-11-26T09:56:18,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978545 2024-11-26T09:56:18,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978545) roll requested 2024-11-26T09:56:18,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978564 2024-11-26T09:56:18,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978564 2024-11-26T09:56:18,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978564) roll requested 2024-11-26T09:56:18,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978576 2024-11-26T09:56:18,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978576 2024-11-26T09:56:18,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978576) roll requested 2024-11-26T09:56:18,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978586 2024-11-26T09:56:18,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978586 2024-11-26T09:56:18,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978586) roll requested 2024-11-26T09:56:18,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978598 2024-11-26T09:56:18,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978598 2024-11-26T09:56:18,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978598) roll requested 2024-11-26T09:56:18,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978613 2024-11-26T09:56:18,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978613 2024-11-26T09:56:18,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978613) roll requested 2024-11-26T09:56:18,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978624 2024-11-26T09:56:18,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978624 2024-11-26T09:56:18,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978624) roll requested 2024-11-26T09:56:18,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978640 2024-11-26T09:56:18,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978640 2024-11-26T09:56:18,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978640) roll requested 2024-11-26T09:56:18,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978656 2024-11-26T09:56:18,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978656 2024-11-26T09:56:18,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978656) roll requested 2024-11-26T09:56:18,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978672 2024-11-26T09:56:18,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978672 2024-11-26T09:56:18,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978672) roll requested 2024-11-26T09:56:18,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978681 2024-11-26T09:56:18,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978681 2024-11-26T09:56:18,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978681) roll requested 2024-11-26T09:56:18,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978692 2024-11-26T09:56:18,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978692 2024-11-26T09:56:18,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978692) roll requested 2024-11-26T09:56:18,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978704 2024-11-26T09:56:18,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978704 2024-11-26T09:56:18,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978704) roll requested 2024-11-26T09:56:18,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978725 2024-11-26T09:56:18,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978725 2024-11-26T09:56:18,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978725) roll requested 2024-11-26T09:56:18,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978738 2024-11-26T09:56:18,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978738 2024-11-26T09:56:18,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978738) roll requested 2024-11-26T09:56:18,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978752 2024-11-26T09:56:18,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978752 2024-11-26T09:56:18,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978752) roll requested 2024-11-26T09:56:18,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978760 2024-11-26T09:56:18,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978760 2024-11-26T09:56:18,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978760) roll requested 2024-11-26T09:56:18,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978769 2024-11-26T09:56:18,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978769 2024-11-26T09:56:18,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978769) roll requested 2024-11-26T09:56:18,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978779 2024-11-26T09:56:18,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978779 2024-11-26T09:56:18,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978779) roll requested 2024-11-26T09:56:18,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978792 2024-11-26T09:56:18,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978792 2024-11-26T09:56:18,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978792) roll requested 2024-11-26T09:56:18,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978801 2024-11-26T09:56:18,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978801 2024-11-26T09:56:18,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978801) roll requested 2024-11-26T09:56:18,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978808 2024-11-26T09:56:18,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978808 2024-11-26T09:56:18,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978808) roll requested 2024-11-26T09:56:18,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978815 2024-11-26T09:56:18,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978815 2024-11-26T09:56:18,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978815) roll requested 2024-11-26T09:56:18,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978828 2024-11-26T09:56:18,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978828 2024-11-26T09:56:18,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978828) roll requested 2024-11-26T09:56:18,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978840 2024-11-26T09:56:18,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978840 2024-11-26T09:56:18,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978840) roll requested 2024-11-26T09:56:18,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978849 2024-11-26T09:56:18,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978849 2024-11-26T09:56:18,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978849) roll requested 2024-11-26T09:56:18,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978856 2024-11-26T09:56:18,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978856 2024-11-26T09:56:18,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978856) roll requested 2024-11-26T09:56:18,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978868 2024-11-26T09:56:18,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978868 2024-11-26T09:56:18,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978868) roll requested 2024-11-26T09:56:18,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978876 2024-11-26T09:56:18,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978876 2024-11-26T09:56:18,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978876) roll requested 2024-11-26T09:56:18,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978892 2024-11-26T09:56:18,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978892 2024-11-26T09:56:18,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978892) roll requested 2024-11-26T09:56:18,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978904 2024-11-26T09:56:18,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978904 2024-11-26T09:56:18,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978904) roll requested 2024-11-26T09:56:18,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978920 2024-11-26T09:56:18,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978920 2024-11-26T09:56:18,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978920) roll requested 2024-11-26T09:56:18,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978933 2024-11-26T09:56:18,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978933 2024-11-26T09:56:18,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978933) roll requested 2024-11-26T09:56:18,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978944 2024-11-26T09:56:18,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978944 2024-11-26T09:56:18,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978944) roll requested 2024-11-26T09:56:18,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978960 2024-11-26T09:56:18,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978960 2024-11-26T09:56:18,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978960) roll requested 2024-11-26T09:56:18,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978976 2024-11-26T09:56:18,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978976 2024-11-26T09:56:18,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:18,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:18,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978976) roll requested 2024-11-26T09:56:18,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614978988 2024-11-26T09:56:18,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:18,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978988 2024-11-26T09:56:19,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614978988) roll requested 2024-11-26T09:56:19,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979000 2024-11-26T09:56:19,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614978988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979000 2024-11-26T09:56:19,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979000) roll requested 2024-11-26T09:56:19,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979019 2024-11-26T09:56:19,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979019 2024-11-26T09:56:19,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979019) roll requested 2024-11-26T09:56:19,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979032 2024-11-26T09:56:19,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979032 2024-11-26T09:56:19,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979032) roll requested 2024-11-26T09:56:19,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979048 2024-11-26T09:56:19,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979048 2024-11-26T09:56:19,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979048) roll requested 2024-11-26T09:56:19,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979068 2024-11-26T09:56:19,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979068 2024-11-26T09:56:19,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979068) roll requested 2024-11-26T09:56:19,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979076 2024-11-26T09:56:19,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979076 2024-11-26T09:56:19,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979076) roll requested 2024-11-26T09:56:19,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979092 2024-11-26T09:56:19,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979092 2024-11-26T09:56:19,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979092) roll requested 2024-11-26T09:56:19,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979101 2024-11-26T09:56:19,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979101 2024-11-26T09:56:19,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979101) roll requested 2024-11-26T09:56:19,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979117 2024-11-26T09:56:19,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979117 2024-11-26T09:56:19,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979117) roll requested 2024-11-26T09:56:19,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979132 2024-11-26T09:56:19,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979132 2024-11-26T09:56:19,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979132) roll requested 2024-11-26T09:56:19,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979145 2024-11-26T09:56:19,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979132 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979145 2024-11-26T09:56:19,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979145) roll requested 2024-11-26T09:56:19,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979155 2024-11-26T09:56:19,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979155 2024-11-26T09:56:19,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979155) roll requested 2024-11-26T09:56:19,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979168 2024-11-26T09:56:19,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979168 2024-11-26T09:56:19,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979168) roll requested 2024-11-26T09:56:19,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979186 2024-11-26T09:56:19,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979186 2024-11-26T09:56:19,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979186) roll requested 2024-11-26T09:56:19,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979200 2024-11-26T09:56:19,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979200 2024-11-26T09:56:19,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979200) roll requested 2024-11-26T09:56:19,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979216 2024-11-26T09:56:19,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979216 2024-11-26T09:56:19,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979216) roll requested 2024-11-26T09:56:19,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979228 2024-11-26T09:56:19,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979228 2024-11-26T09:56:19,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979228) roll requested 2024-11-26T09:56:19,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979240 2024-11-26T09:56:19,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979240 2024-11-26T09:56:19,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979240) roll requested 2024-11-26T09:56:19,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979246 2024-11-26T09:56:19,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979246 2024-11-26T09:56:19,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979246) roll requested 2024-11-26T09:56:19,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979253 2024-11-26T09:56:19,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979253 2024-11-26T09:56:19,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979253) roll requested 2024-11-26T09:56:19,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979260 2024-11-26T09:56:19,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979260 2024-11-26T09:56:19,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979260) roll requested 2024-11-26T09:56:19,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979271 2024-11-26T09:56:19,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979271 2024-11-26T09:56:19,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979271) roll requested 2024-11-26T09:56:19,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979284 2024-11-26T09:56:19,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979284 2024-11-26T09:56:19,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979284) roll requested 2024-11-26T09:56:19,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979300 2024-11-26T09:56:19,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979300 2024-11-26T09:56:19,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979300) roll requested 2024-11-26T09:56:19,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979312 2024-11-26T09:56:19,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979312 2024-11-26T09:56:19,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979312) roll requested 2024-11-26T09:56:19,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979328 2024-11-26T09:56:19,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979328 2024-11-26T09:56:19,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979328) roll requested 2024-11-26T09:56:19,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979335 2024-11-26T09:56:19,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979335 2024-11-26T09:56:19,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979335) roll requested 2024-11-26T09:56:19,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979344 2024-11-26T09:56:19,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979344 2024-11-26T09:56:19,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979344) roll requested 2024-11-26T09:56:19,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979356 2024-11-26T09:56:19,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979356 2024-11-26T09:56:19,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979356) roll requested 2024-11-26T09:56:19,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979363 2024-11-26T09:56:19,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979363 2024-11-26T09:56:19,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979363) roll requested 2024-11-26T09:56:19,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979372 2024-11-26T09:56:19,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979372 2024-11-26T09:56:19,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979372) roll requested 2024-11-26T09:56:19,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979382 2024-11-26T09:56:19,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979382 2024-11-26T09:56:19,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979382) roll requested 2024-11-26T09:56:19,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979392 2024-11-26T09:56:19,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979392 2024-11-26T09:56:19,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979392) roll requested 2024-11-26T09:56:19,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979404 2024-11-26T09:56:19,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979404 2024-11-26T09:56:19,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979404) roll requested 2024-11-26T09:56:19,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979413 2024-11-26T09:56:19,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979413 2024-11-26T09:56:19,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979413) roll requested 2024-11-26T09:56:19,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979424 2024-11-26T09:56:19,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979424 2024-11-26T09:56:19,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979424) roll requested 2024-11-26T09:56:19,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979437 2024-11-26T09:56:19,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979437 2024-11-26T09:56:19,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979437) roll requested 2024-11-26T09:56:19,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979448 2024-11-26T09:56:19,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979448 2024-11-26T09:56:19,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979448) roll requested 2024-11-26T09:56:19,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979460 2024-11-26T09:56:19,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979460 2024-11-26T09:56:19,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979460) roll requested 2024-11-26T09:56:19,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979468 2024-11-26T09:56:19,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979468 2024-11-26T09:56:19,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979468) roll requested 2024-11-26T09:56:19,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979476 2024-11-26T09:56:19,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979476 2024-11-26T09:56:19,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979476) roll requested 2024-11-26T09:56:19,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979484 2024-11-26T09:56:19,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979484 2024-11-26T09:56:19,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979484) roll requested 2024-11-26T09:56:19,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979492 2024-11-26T09:56:19,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979492 2024-11-26T09:56:19,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979492) roll requested 2024-11-26T09:56:19,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979500 2024-11-26T09:56:19,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979500 2024-11-26T09:56:19,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979500) roll requested 2024-11-26T09:56:19,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979507 2024-11-26T09:56:19,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979507 2024-11-26T09:56:19,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979507) roll requested 2024-11-26T09:56:19,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979516 2024-11-26T09:56:19,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979516 2024-11-26T09:56:19,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:19,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979516) roll requested 2024-11-26T09:56:19,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979524 2024-11-26T09:56:19,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979524 2024-11-26T09:56:19,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,532 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979524) roll requested 2024-11-26T09:56:19,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979533 2024-11-26T09:56:19,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979524 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979533 2024-11-26T09:56:19,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979533) roll requested 2024-11-26T09:56:19,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979544 2024-11-26T09:56:19,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979533 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979544 2024-11-26T09:56:19,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,560 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979544) roll requested 2024-11-26T09:56:19,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979561 2024-11-26T09:56:19,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979544 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979561 2024-11-26T09:56:19,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979561) roll requested 2024-11-26T09:56:19,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979572 2024-11-26T09:56:19,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979561 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979572 2024-11-26T09:56:19,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,588 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979572) roll requested 2024-11-26T09:56:19,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979589 2024-11-26T09:56:19,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979572 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979589 2024-11-26T09:56:19,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,600 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979589) roll requested 2024-11-26T09:56:19,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979600 2024-11-26T09:56:19,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979600 2024-11-26T09:56:19,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979600) roll requested 2024-11-26T09:56:19,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979609 2024-11-26T09:56:19,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979600 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979609 2024-11-26T09:56:19,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,622 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979609) roll requested 2024-11-26T09:56:19,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979622 2024-11-26T09:56:19,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979622 2024-11-26T09:56:19,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,636 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979622) roll requested 2024-11-26T09:56:19,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979637 2024-11-26T09:56:19,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979622 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979637 2024-11-26T09:56:19,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,648 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979637) roll requested 2024-11-26T09:56:19,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979649 2024-11-26T09:56:19,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979637 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979649 2024-11-26T09:56:19,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,667 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979649) roll requested 2024-11-26T09:56:19,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979667 2024-11-26T09:56:19,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979649 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979667 2024-11-26T09:56:19,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979667) roll requested 2024-11-26T09:56:19,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979680 2024-11-26T09:56:19,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979667 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979680 2024-11-26T09:56:19,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,691 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979680) roll requested 2024-11-26T09:56:19,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979691 2024-11-26T09:56:19,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979680 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979691 2024-11-26T09:56:19,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979691) roll requested 2024-11-26T09:56:19,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979700 2024-11-26T09:56:19,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979700 2024-11-26T09:56:19,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,710 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979700) roll requested 2024-11-26T09:56:19,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979710 2024-11-26T09:56:19,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979700 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979710 2024-11-26T09:56:19,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,717 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979710) roll requested 2024-11-26T09:56:19,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979717 2024-11-26T09:56:19,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979710 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979717 2024-11-26T09:56:19,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979717) roll requested 2024-11-26T09:56:19,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979728 2024-11-26T09:56:19,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979728 2024-11-26T09:56:19,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979728) roll requested 2024-11-26T09:56:19,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979741 2024-11-26T09:56:19,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979728 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979741 2024-11-26T09:56:19,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979741) roll requested 2024-11-26T09:56:19,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979757 2024-11-26T09:56:19,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979757 2024-11-26T09:56:19,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,764 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979757) roll requested 2024-11-26T09:56:19,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979764 2024-11-26T09:56:19,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979764 2024-11-26T09:56:19,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,776 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979764) roll requested 2024-11-26T09:56:19,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979777 2024-11-26T09:56:19,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979764 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979777 2024-11-26T09:56:19,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,789 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979777) roll requested 2024-11-26T09:56:19,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979790 2024-11-26T09:56:19,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979790 2024-11-26T09:56:19,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,798 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979790) roll requested 2024-11-26T09:56:19,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979798 2024-11-26T09:56:19,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979790 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979798 2024-11-26T09:56:19,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979798) roll requested 2024-11-26T09:56:19,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979813 2024-11-26T09:56:19,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979798 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979813 2024-11-26T09:56:19,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,832 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979813) roll requested 2024-11-26T09:56:19,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979832 2024-11-26T09:56:19,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979832 2024-11-26T09:56:19,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979832) roll requested 2024-11-26T09:56:19,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979840 2024-11-26T09:56:19,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979832 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979840 2024-11-26T09:56:19,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,848 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979840) roll requested 2024-11-26T09:56:19,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979848 2024-11-26T09:56:19,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979840 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979848 2024-11-26T09:56:19,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,859 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979848) roll requested 2024-11-26T09:56:19,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979860 2024-11-26T09:56:19,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979848 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979860 2024-11-26T09:56:19,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,869 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979860) roll requested 2024-11-26T09:56:19,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979869 2024-11-26T09:56:19,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979860 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979869 2024-11-26T09:56:19,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979869) roll requested 2024-11-26T09:56:19,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979880 2024-11-26T09:56:19,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979869 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979880 2024-11-26T09:56:19,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979880) roll requested 2024-11-26T09:56:19,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979887 2024-11-26T09:56:19,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979887 2024-11-26T09:56:19,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,896 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979887) roll requested 2024-11-26T09:56:19,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979896 2024-11-26T09:56:19,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979887 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979896 2024-11-26T09:56:19,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,903 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979896) roll requested 2024-11-26T09:56:19,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979903 2024-11-26T09:56:19,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979896 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979903 2024-11-26T09:56:19,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,910 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979903) roll requested 2024-11-26T09:56:19,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979911 2024-11-26T09:56:19,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979903 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979911 2024-11-26T09:56:19,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979911) roll requested 2024-11-26T09:56:19,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979920 2024-11-26T09:56:19,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979911 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979920 2024-11-26T09:56:19,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,928 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979920) roll requested 2024-11-26T09:56:19,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979928 2024-11-26T09:56:19,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979928 2024-11-26T09:56:19,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,936 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979928) roll requested 2024-11-26T09:56:19,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979936 2024-11-26T09:56:19,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979936 2024-11-26T09:56:19,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,943 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979936) roll requested 2024-11-26T09:56:19,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979943 2024-11-26T09:56:19,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979936 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979943 2024-11-26T09:56:19,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979943) roll requested 2024-11-26T09:56:19,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979952 2024-11-26T09:56:19,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979943 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979952 2024-11-26T09:56:19,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979952) roll requested 2024-11-26T09:56:19,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979960 2024-11-26T09:56:19,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979960 2024-11-26T09:56:19,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,968 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979960) roll requested 2024-11-26T09:56:19,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979968 2024-11-26T09:56:19,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979968 2024-11-26T09:56:19,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,975 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979968) roll requested 2024-11-26T09:56:19,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979975 2024-11-26T09:56:19,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979968 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979975 2024-11-26T09:56:19,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,982 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979975) roll requested 2024-11-26T09:56:19,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979983 2024-11-26T09:56:19,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979975 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979983 2024-11-26T09:56:19,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,989 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979983) roll requested 2024-11-26T09:56:19,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979990 2024-11-26T09:56:19,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:19,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979983 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979990 2024-11-26T09:56:19,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:19,998 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:19,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979990) roll requested 2024-11-26T09:56:19,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614979999 2024-11-26T09:56:20,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979990 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979999 2024-11-26T09:56:20,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,008 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614979999) roll requested 2024-11-26T09:56:20,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980008 2024-11-26T09:56:20,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614979999 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980008 2024-11-26T09:56:20,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980008) roll requested 2024-11-26T09:56:20,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980024 2024-11-26T09:56:20,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980008 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980024 2024-11-26T09:56:20,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,035 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980024) roll requested 2024-11-26T09:56:20,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980036 2024-11-26T09:56:20,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980024 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980036 2024-11-26T09:56:20,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,050 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980036) roll requested 2024-11-26T09:56:20,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980050 2024-11-26T09:56:20,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980036 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980050 2024-11-26T09:56:20,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980050) roll requested 2024-11-26T09:56:20,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980060 2024-11-26T09:56:20,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980050 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980060 2024-11-26T09:56:20,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,069 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980060) roll requested 2024-11-26T09:56:20,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980069 2024-11-26T09:56:20,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980060 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980069 2024-11-26T09:56:20,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,080 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980069) roll requested 2024-11-26T09:56:20,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980080 2024-11-26T09:56:20,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980069 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980080 2024-11-26T09:56:20,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980080) roll requested 2024-11-26T09:56:20,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980092 2024-11-26T09:56:20,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980080 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980092 2024-11-26T09:56:20,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,105 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980092) roll requested 2024-11-26T09:56:20,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980105 2024-11-26T09:56:20,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980092 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980105 2024-11-26T09:56:20,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980105) roll requested 2024-11-26T09:56:20,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980121 2024-11-26T09:56:20,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980121 2024-11-26T09:56:20,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,131 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,132 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=56, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,133 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980121) roll requested 2024-11-26T09:56:20,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980133 2024-11-26T09:56:20,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980133 2024-11-26T09:56:20,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,144 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=57, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,144 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,144 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980133) roll requested 2024-11-26T09:56:20,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980144 2024-11-26T09:56:20,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980144 2024-11-26T09:56:20,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,152 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=58, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,152 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,152 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980144) roll requested 2024-11-26T09:56:20,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980152 2024-11-26T09:56:20,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980144 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980152 2024-11-26T09:56:20,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,160 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=59, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,160 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,160 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980152) roll requested 2024-11-26T09:56:20,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980160 2024-11-26T09:56:20,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980152 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980160 2024-11-26T09:56:20,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,167 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=60, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,167 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,167 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980160) roll requested 2024-11-26T09:56:20,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980167 2024-11-26T09:56:20,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980160 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980167 2024-11-26T09:56:20,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,175 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=61, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,175 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,175 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980167) roll requested 2024-11-26T09:56:20,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980176 2024-11-26T09:56:20,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980167 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980176 2024-11-26T09:56:20,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,183 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=62, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,183 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,183 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980176) roll requested 2024-11-26T09:56:20,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980183 2024-11-26T09:56:20,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980176 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980183 2024-11-26T09:56:20,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=63, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,198 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980183) roll requested 2024-11-26T09:56:20,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980198 2024-11-26T09:56:20,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980183 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980198 2024-11-26T09:56:20,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,208 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=64, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,208 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,208 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980198) roll requested 2024-11-26T09:56:20,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980209 2024-11-26T09:56:20,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980198 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980209 2024-11-26T09:56:20,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,227 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=65, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,227 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,227 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980209) roll requested 2024-11-26T09:56:20,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980227 2024-11-26T09:56:20,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980227 2024-11-26T09:56:20,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,242 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=66, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,243 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,243 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980227) roll requested 2024-11-26T09:56:20,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980243 2024-11-26T09:56:20,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980227 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980243 2024-11-26T09:56:20,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=67, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,256 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980243) roll requested 2024-11-26T09:56:20,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980257 2024-11-26T09:56:20,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980243 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980257 2024-11-26T09:56:20,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,265 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=68, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,265 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,266 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980257) roll requested 2024-11-26T09:56:20,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980266 2024-11-26T09:56:20,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980266 2024-11-26T09:56:20,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,273 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=69, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,273 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,273 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980266) roll requested 2024-11-26T09:56:20,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980273 2024-11-26T09:56:20,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980266 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980273 2024-11-26T09:56:20,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,281 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=70, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,281 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,281 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980273) roll requested 2024-11-26T09:56:20,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980281 2024-11-26T09:56:20,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980273 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980281 2024-11-26T09:56:20,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=71, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,297 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980281) roll requested 2024-11-26T09:56:20,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980297 2024-11-26T09:56:20,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980281 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980297 2024-11-26T09:56:20,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,308 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=72, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,308 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980297) roll requested 2024-11-26T09:56:20,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980308 2024-11-26T09:56:20,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980308 2024-11-26T09:56:20,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,319 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=73, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980308) roll requested 2024-11-26T09:56:20,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980320 2024-11-26T09:56:20,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980308 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980320 2024-11-26T09:56:20,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=74, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,329 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980320) roll requested 2024-11-26T09:56:20,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980329 2024-11-26T09:56:20,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980320 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980329 2024-11-26T09:56:20,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=75, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,336 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980329) roll requested 2024-11-26T09:56:20,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980336 2024-11-26T09:56:20,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980336 2024-11-26T09:56:20,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,344 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=76, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,344 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980336) roll requested 2024-11-26T09:56:20,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980344 2024-11-26T09:56:20,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980336 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980344 2024-11-26T09:56:20,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,356 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=77, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,356 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,356 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980344) roll requested 2024-11-26T09:56:20,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980357 2024-11-26T09:56:20,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980344 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980357 2024-11-26T09:56:20,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,369 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=78, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,370 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980357) roll requested 2024-11-26T09:56:20,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980370 2024-11-26T09:56:20,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980357 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980370 2024-11-26T09:56:20,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=79, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980370) roll requested 2024-11-26T09:56:20,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980380 2024-11-26T09:56:20,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980380 2024-11-26T09:56:20,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,387 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=80, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,387 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,387 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980380) roll requested 2024-11-26T09:56:20,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980387 2024-11-26T09:56:20,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980387 2024-11-26T09:56:20,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,395 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=81, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,395 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,395 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980387) roll requested 2024-11-26T09:56:20,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980395 2024-11-26T09:56:20,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980387 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980395 2024-11-26T09:56:20,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,404 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=82, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,404 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,404 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980395) roll requested 2024-11-26T09:56:20,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980405 2024-11-26T09:56:20,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980395 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980405 2024-11-26T09:56:20,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=83, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980405) roll requested 2024-11-26T09:56:20,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980420 2024-11-26T09:56:20,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980405 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980420 2024-11-26T09:56:20,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,432 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=84, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,432 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980420) roll requested 2024-11-26T09:56:20,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980432 2024-11-26T09:56:20,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980420 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980432 2024-11-26T09:56:20,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,448 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=85, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,448 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,448 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980432) roll requested 2024-11-26T09:56:20,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980449 2024-11-26T09:56:20,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980432 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980449 2024-11-26T09:56:20,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,459 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=86, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,459 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,459 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980449) roll requested 2024-11-26T09:56:20,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980460 2024-11-26T09:56:20,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980449 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980460 2024-11-26T09:56:20,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=87, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,467 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,467 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980460) roll requested 2024-11-26T09:56:20,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980467 2024-11-26T09:56:20,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980460 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980467 2024-11-26T09:56:20,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,474 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=88, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,474 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980467) roll requested 2024-11-26T09:56:20,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980474 2024-11-26T09:56:20,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980467 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980474 2024-11-26T09:56:20,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=89, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980474) roll requested 2024-11-26T09:56:20,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980485 2024-11-26T09:56:20,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980474 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980485 2024-11-26T09:56:20,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=90, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,493 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980485) roll requested 2024-11-26T09:56:20,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980493 2024-11-26T09:56:20,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980493 2024-11-26T09:56:20,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,500 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=91, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,500 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,500 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980493) roll requested 2024-11-26T09:56:20,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980500 2024-11-26T09:56:20,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980500 2024-11-26T09:56:20,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,507 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=92, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,507 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,507 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980500) roll requested 2024-11-26T09:56:20,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980508 2024-11-26T09:56:20,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980500 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980508 2024-11-26T09:56:20,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,514 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=93, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,514 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,514 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980508) roll requested 2024-11-26T09:56:20,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980515 2024-11-26T09:56:20,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980508 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980515 2024-11-26T09:56:20,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,527 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=94, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,527 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,527 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980515) roll requested 2024-11-26T09:56:20,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980527 2024-11-26T09:56:20,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980515 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980527 2024-11-26T09:56:20,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,541 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=95, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,542 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980527) roll requested 2024-11-26T09:56:20,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980542 2024-11-26T09:56:20,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980527 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980542 2024-11-26T09:56:20,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,551 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=96, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,551 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,551 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980542) roll requested 2024-11-26T09:56:20,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980552 2024-11-26T09:56:20,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980542 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980552 2024-11-26T09:56:20,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,568 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=97, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,568 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980552) roll requested 2024-11-26T09:56:20,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980568 2024-11-26T09:56:20,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980552 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980568 2024-11-26T09:56:20,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,576 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=98, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,576 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,576 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980568) roll requested 2024-11-26T09:56:20,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980577 2024-11-26T09:56:20,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980568 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980577 2024-11-26T09:56:20,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,591 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=99, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,591 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,591 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980577) roll requested 2024-11-26T09:56:20,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980591 2024-11-26T09:56:20,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980577 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980591 2024-11-26T09:56:20,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,604 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=100, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,604 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980591) roll requested 2024-11-26T09:56:20,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980604 2024-11-26T09:56:20,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980591 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980604 2024-11-26T09:56:20,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,616 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=101, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,616 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,617 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980604) roll requested 2024-11-26T09:56:20,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980617 2024-11-26T09:56:20,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980604 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980617 2024-11-26T09:56:20,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,628 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=102, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980617) roll requested 2024-11-26T09:56:20,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980628 2024-11-26T09:56:20,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980628 2024-11-26T09:56:20,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,637 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=103, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,637 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980628) roll requested 2024-11-26T09:56:20,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980638 2024-11-26T09:56:20,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980638 2024-11-26T09:56:20,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=104, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980638) roll requested 2024-11-26T09:56:20,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980651 2024-11-26T09:56:20,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980638 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980651 2024-11-26T09:56:20,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,660 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=105, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,660 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,660 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980651) roll requested 2024-11-26T09:56:20,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980660 2024-11-26T09:56:20,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980660 2024-11-26T09:56:20,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,676 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=106, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,676 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,676 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980660) roll requested 2024-11-26T09:56:20,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980676 2024-11-26T09:56:20,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980660 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980676 2024-11-26T09:56:20,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,684 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=107, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,684 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,684 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980676) roll requested 2024-11-26T09:56:20,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980685 2024-11-26T09:56:20,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980676 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980685 2024-11-26T09:56:20,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,696 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=108, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,696 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,696 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980685) roll requested 2024-11-26T09:56:20,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980696 2024-11-26T09:56:20,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980685 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980696 2024-11-26T09:56:20,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,704 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=109, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,704 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,704 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980696) roll requested 2024-11-26T09:56:20,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980704 2024-11-26T09:56:20,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980704 2024-11-26T09:56:20,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,713 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=110, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,713 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,714 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980704) roll requested 2024-11-26T09:56:20,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980714 2024-11-26T09:56:20,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980704 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980714 2024-11-26T09:56:20,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,726 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=111, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,726 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980714) roll requested 2024-11-26T09:56:20,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980726 2024-11-26T09:56:20,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980714 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980726 2024-11-26T09:56:20,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,739 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=112, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,739 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,739 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980726) roll requested 2024-11-26T09:56:20,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980740 2024-11-26T09:56:20,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980726 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980740 2024-11-26T09:56:20,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=113, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,748 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980740) roll requested 2024-11-26T09:56:20,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980748 2024-11-26T09:56:20,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980740 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980748 2024-11-26T09:56:20,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,768 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=114, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,768 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980748) roll requested 2024-11-26T09:56:20,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980768 2024-11-26T09:56:20,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980748 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980768 2024-11-26T09:56:20,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,781 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=115, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,781 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,781 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980768) roll requested 2024-11-26T09:56:20,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980781 2024-11-26T09:56:20,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980768 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980781 2024-11-26T09:56:20,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,790 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=116, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,790 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,790 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980781) roll requested 2024-11-26T09:56:20,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980790 2024-11-26T09:56:20,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980781 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980790 2024-11-26T09:56:20,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,804 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=117, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,804 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,804 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980790) roll requested 2024-11-26T09:56:20,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980804 2024-11-26T09:56:20,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980790 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980804 2024-11-26T09:56:20,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,816 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=118, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,816 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,816 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980804) roll requested 2024-11-26T09:56:20,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980816 2024-11-26T09:56:20,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980804 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980816 2024-11-26T09:56:20,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,827 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=119, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980816) roll requested 2024-11-26T09:56:20,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980828 2024-11-26T09:56:20,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980816 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980828 2024-11-26T09:56:20,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,840 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=120, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,840 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980828) roll requested 2024-11-26T09:56:20,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980840 2024-11-26T09:56:20,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980840 2024-11-26T09:56:20,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,847 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=121, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,847 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980840) roll requested 2024-11-26T09:56:20,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980847 2024-11-26T09:56:20,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980840 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980847 2024-11-26T09:56:20,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,854 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=122, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,854 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980847) roll requested 2024-11-26T09:56:20,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980854 2024-11-26T09:56:20,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980847 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980854 2024-11-26T09:56:20,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,871 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=123, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,871 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980854) roll requested 2024-11-26T09:56:20,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980871 2024-11-26T09:56:20,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980871 2024-11-26T09:56:20,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,878 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=124, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,878 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980871) roll requested 2024-11-26T09:56:20,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980879 2024-11-26T09:56:20,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980871 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980879 2024-11-26T09:56:20,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,885 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=125, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,885 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,886 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980879) roll requested 2024-11-26T09:56:20,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980886 2024-11-26T09:56:20,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980879 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980886 2024-11-26T09:56:20,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,897 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=126, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,897 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,897 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980886) roll requested 2024-11-26T09:56:20,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980897 2024-11-26T09:56:20,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980886 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980897 2024-11-26T09:56:20,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,904 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=127, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,904 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,904 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980897) roll requested 2024-11-26T09:56:20,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980905 2024-11-26T09:56:20,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980905 2024-11-26T09:56:20,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,912 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=128, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,912 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,912 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980905) roll requested 2024-11-26T09:56:20,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980913 2024-11-26T09:56:20,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980905 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980913 2024-11-26T09:56:20,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=129, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,920 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980913) roll requested 2024-11-26T09:56:20,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980920 2024-11-26T09:56:20,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980913 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980920 2024-11-26T09:56:20,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,932 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=130, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,932 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980920) roll requested 2024-11-26T09:56:20,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980932 2024-11-26T09:56:20,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980932 2024-11-26T09:56:20,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,939 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=131, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,939 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,939 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980932) roll requested 2024-11-26T09:56:20,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980940 2024-11-26T09:56:20,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980932 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980940 2024-11-26T09:56:20,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,953 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=132, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,953 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,953 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980940) roll requested 2024-11-26T09:56:20,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980953 2024-11-26T09:56:20,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980940 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980953 2024-11-26T09:56:20,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,963 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=133, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,963 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,963 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980953) roll requested 2024-11-26T09:56:20,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980964 2024-11-26T09:56:20,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980964 2024-11-26T09:56:20,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=134, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,977 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980964) roll requested 2024-11-26T09:56:20,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980977 2024-11-26T09:56:20,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:20,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980977 2024-11-26T09:56:20,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:20,992 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=135, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:20,992 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,992 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:20,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:20,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980977) roll requested 2024-11-26T09:56:20,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614980992 2024-11-26T09:56:21,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980977 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980992 2024-11-26T09:56:21,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,009 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,009 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=136, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,010 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614980992) roll requested 2024-11-26T09:56:21,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981010 2024-11-26T09:56:21,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614980992 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981010 2024-11-26T09:56:21,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=137, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,018 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981010) roll requested 2024-11-26T09:56:21,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981018 2024-11-26T09:56:21,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981010 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981018 2024-11-26T09:56:21,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,026 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,026 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=138, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,026 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981018) roll requested 2024-11-26T09:56:21,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981026 2024-11-26T09:56:21,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981018 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981026 2024-11-26T09:56:21,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,035 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=139, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,035 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,035 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981026) roll requested 2024-11-26T09:56:21,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981035 2024-11-26T09:56:21,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981026 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981035 2024-11-26T09:56:21,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,044 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=140, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,044 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981035) roll requested 2024-11-26T09:56:21,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981044 2024-11-26T09:56:21,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981035 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981044 2024-11-26T09:56:21,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,052 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=141, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,052 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,052 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981044) roll requested 2024-11-26T09:56:21,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981052 2024-11-26T09:56:21,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981044 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981052 2024-11-26T09:56:21,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,060 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=142, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,060 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981052) roll requested 2024-11-26T09:56:21,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981060 2024-11-26T09:56:21,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981052 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981060 2024-11-26T09:56:21,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,067 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=143, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,067 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,067 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981060) roll requested 2024-11-26T09:56:21,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981067 2024-11-26T09:56:21,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981060 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981067 2024-11-26T09:56:21,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,075 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=144, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,075 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,075 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981067) roll requested 2024-11-26T09:56:21,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981076 2024-11-26T09:56:21,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981067 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981076 2024-11-26T09:56:21,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,082 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=145, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,082 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,082 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981076) roll requested 2024-11-26T09:56:21,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981082 2024-11-26T09:56:21,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981076 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981082 2024-11-26T09:56:21,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,090 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=146, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,090 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,090 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981082) roll requested 2024-11-26T09:56:21,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981091 2024-11-26T09:56:21,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981082 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981091 2024-11-26T09:56:21,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=147, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981091) roll requested 2024-11-26T09:56:21,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981097 2024-11-26T09:56:21,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981091 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981097 2024-11-26T09:56:21,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,106 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=148, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,106 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,106 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981097) roll requested 2024-11-26T09:56:21,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981106 2024-11-26T09:56:21,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981106 2024-11-26T09:56:21,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,112 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=149, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,113 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981106) roll requested 2024-11-26T09:56:21,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981113 2024-11-26T09:56:21,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981106 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981113 2024-11-26T09:56:21,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,120 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=150, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,120 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981113) roll requested 2024-11-26T09:56:21,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981120 2024-11-26T09:56:21,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981120 2024-11-26T09:56:21,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,127 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=151, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,127 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,127 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981120) roll requested 2024-11-26T09:56:21,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981127 2024-11-26T09:56:21,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981120 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981127 2024-11-26T09:56:21,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=152, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,136 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,136 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981127) roll requested 2024-11-26T09:56:21,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981136 2024-11-26T09:56:21,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981127 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981136 2024-11-26T09:56:21,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,143 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=153, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,143 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,143 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981136) roll requested 2024-11-26T09:56:21,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981143 2024-11-26T09:56:21,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981136 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981143 2024-11-26T09:56:21,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,156 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=154, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,156 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981143) roll requested 2024-11-26T09:56:21,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981156 2024-11-26T09:56:21,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981143 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981156 2024-11-26T09:56:21,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,164 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=155, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981156) roll requested 2024-11-26T09:56:21,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981165 2024-11-26T09:56:21,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981165 2024-11-26T09:56:21,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,177 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=156, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,177 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,177 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981165) roll requested 2024-11-26T09:56:21,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981177 2024-11-26T09:56:21,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981177 2024-11-26T09:56:21,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=157, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981177) roll requested 2024-11-26T09:56:21,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981184 2024-11-26T09:56:21,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981184 2024-11-26T09:56:21,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,194 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=158, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,194 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,194 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981184) roll requested 2024-11-26T09:56:21,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981194 2024-11-26T09:56:21,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981184 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981194 2024-11-26T09:56:21,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,208 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=159, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,209 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981194) roll requested 2024-11-26T09:56:21,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981209 2024-11-26T09:56:21,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981194 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981209 2024-11-26T09:56:21,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=160, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981209) roll requested 2024-11-26T09:56:21,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981217 2024-11-26T09:56:21,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981217 2024-11-26T09:56:21,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,227 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=161, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,227 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,227 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981217) roll requested 2024-11-26T09:56:21,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981227 2024-11-26T09:56:21,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981227 2024-11-26T09:56:21,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,234 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=162, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,234 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,234 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981227) roll requested 2024-11-26T09:56:21,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981234 2024-11-26T09:56:21,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981227 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981234 2024-11-26T09:56:21,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,245 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=163, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,246 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,246 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981234) roll requested 2024-11-26T09:56:21,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981246 2024-11-26T09:56:21,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981234 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981246 2024-11-26T09:56:21,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,254 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=164, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,254 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,255 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981246) roll requested 2024-11-26T09:56:21,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981255 2024-11-26T09:56:21,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981246 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981255 2024-11-26T09:56:21,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,264 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=165, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,264 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,264 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981255) roll requested 2024-11-26T09:56:21,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981264 2024-11-26T09:56:21,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981255 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981264 2024-11-26T09:56:21,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=166, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,272 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981264) roll requested 2024-11-26T09:56:21,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981272 2024-11-26T09:56:21,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981272 2024-11-26T09:56:21,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,280 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=167, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,280 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,280 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981272) roll requested 2024-11-26T09:56:21,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981280 2024-11-26T09:56:21,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981280 2024-11-26T09:56:21,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=168, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981280) roll requested 2024-11-26T09:56:21,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981292 2024-11-26T09:56:21,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981280 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981292 2024-11-26T09:56:21,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,299 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=169, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,299 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,300 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981292) roll requested 2024-11-26T09:56:21,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981300 2024-11-26T09:56:21,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981300 2024-11-26T09:56:21,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,307 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=170, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,307 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,307 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981300) roll requested 2024-11-26T09:56:21,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981307 2024-11-26T09:56:21,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981300 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981307 2024-11-26T09:56:21,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,316 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=171, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,316 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981307) roll requested 2024-11-26T09:56:21,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981317 2024-11-26T09:56:21,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981307 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981317 2024-11-26T09:56:21,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,338 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=172, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,338 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,338 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981317) roll requested 2024-11-26T09:56:21,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981338 2024-11-26T09:56:21,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981338 2024-11-26T09:56:21,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,350 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,350 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=173, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,351 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981338) roll requested 2024-11-26T09:56:21,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981351 2024-11-26T09:56:21,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981338 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981351 2024-11-26T09:56:21,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,358 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=174, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,358 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,358 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981351) roll requested 2024-11-26T09:56:21,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981359 2024-11-26T09:56:21,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981351 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981359 2024-11-26T09:56:21,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,376 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=175, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,376 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,376 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981359) roll requested 2024-11-26T09:56:21,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981376 2024-11-26T09:56:21,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981359 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981376 2024-11-26T09:56:21,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,383 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=176, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,383 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,384 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981376) roll requested 2024-11-26T09:56:21,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981384 2024-11-26T09:56:21,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981376 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981384 2024-11-26T09:56:21,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,391 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=177, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,391 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,391 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981384) roll requested 2024-11-26T09:56:21,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981391 2024-11-26T09:56:21,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981384 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981391 2024-11-26T09:56:21,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=178, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981391) roll requested 2024-11-26T09:56:21,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981398 2024-11-26T09:56:21,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981391 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981398 2024-11-26T09:56:21,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,406 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=179, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,406 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,407 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981398) roll requested 2024-11-26T09:56:21,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981407 2024-11-26T09:56:21,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981398 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981407 2024-11-26T09:56:21,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,419 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,419 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=180, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,419 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981407) roll requested 2024-11-26T09:56:21,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981419 2024-11-26T09:56:21,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981407 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981419 2024-11-26T09:56:21,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=181, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,432 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981419) roll requested 2024-11-26T09:56:21,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981432 2024-11-26T09:56:21,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981419 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981432 2024-11-26T09:56:21,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,441 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=182, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,441 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,441 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981432) roll requested 2024-11-26T09:56:21,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981441 2024-11-26T09:56:21,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981432 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981441 2024-11-26T09:56:21,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,452 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=183, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,452 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,453 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981441) roll requested 2024-11-26T09:56:21,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981453 2024-11-26T09:56:21,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981441 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981453 2024-11-26T09:56:21,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,460 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=184, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,460 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981453) roll requested 2024-11-26T09:56:21,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981461 2024-11-26T09:56:21,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981453 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981461 2024-11-26T09:56:21,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=185, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,467 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,467 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981461) roll requested 2024-11-26T09:56:21,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981467 2024-11-26T09:56:21,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981467 2024-11-26T09:56:21,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,475 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,475 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=186, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,475 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981467) roll requested 2024-11-26T09:56:21,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981475 2024-11-26T09:56:21,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981467 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981475 2024-11-26T09:56:21,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,482 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=187, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,482 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,482 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981475) roll requested 2024-11-26T09:56:21,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981482 2024-11-26T09:56:21,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981475 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981482 2024-11-26T09:56:21,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,489 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=188, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,489 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,489 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981482) roll requested 2024-11-26T09:56:21,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981489 2024-11-26T09:56:21,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981482 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981489 2024-11-26T09:56:21,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,497 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=189, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,497 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981489) roll requested 2024-11-26T09:56:21,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981497 2024-11-26T09:56:21,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981489 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981497 2024-11-26T09:56:21,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=190, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,503 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981497) roll requested 2024-11-26T09:56:21,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981503 2024-11-26T09:56:21,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981503 2024-11-26T09:56:21,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,510 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,510 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=191, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,510 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981503) roll requested 2024-11-26T09:56:21,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981510 2024-11-26T09:56:21,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981503 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981510 2024-11-26T09:56:21,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,521 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=192, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,521 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,521 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981510) roll requested 2024-11-26T09:56:21,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981521 2024-11-26T09:56:21,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981510 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981521 2024-11-26T09:56:21,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,528 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=193, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,528 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981521) roll requested 2024-11-26T09:56:21,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981528 2024-11-26T09:56:21,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981528 2024-11-26T09:56:21,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=194, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,537 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,538 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981528) roll requested 2024-11-26T09:56:21,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981538 2024-11-26T09:56:21,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981528 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981538 2024-11-26T09:56:21,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,544 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=195, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,544 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981538) roll requested 2024-11-26T09:56:21,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981544 2024-11-26T09:56:21,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981538 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981544 2024-11-26T09:56:21,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,552 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=196, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,552 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,552 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981544) roll requested 2024-11-26T09:56:21,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981552 2024-11-26T09:56:21,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981544 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981552 2024-11-26T09:56:21,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,562 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=197, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,562 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,562 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981552) roll requested 2024-11-26T09:56:21,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981562 2024-11-26T09:56:21,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981552 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981562 2024-11-26T09:56:21,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,568 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=198, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,569 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981562) roll requested 2024-11-26T09:56:21,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981569 2024-11-26T09:56:21,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981562 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981569 2024-11-26T09:56:21,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,575 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=199, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,575 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,576 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981569) roll requested 2024-11-26T09:56:21,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981576 2024-11-26T09:56:21,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981576 2024-11-26T09:56:21,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,582 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=200, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,582 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981576) roll requested 2024-11-26T09:56:21,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981582 2024-11-26T09:56:21,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981576 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981582 2024-11-26T09:56:21,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=201, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,589 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,589 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981582) roll requested 2024-11-26T09:56:21,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981589 2024-11-26T09:56:21,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981582 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981589 2024-11-26T09:56:21,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,595 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=202, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,595 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,595 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981589) roll requested 2024-11-26T09:56:21,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981595 2024-11-26T09:56:21,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981595 2024-11-26T09:56:21,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,603 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,603 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=203, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,603 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981595) roll requested 2024-11-26T09:56:21,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981603 2024-11-26T09:56:21,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981595 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981603 2024-11-26T09:56:21,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,610 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=204, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,610 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,610 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981603) roll requested 2024-11-26T09:56:21,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981610 2024-11-26T09:56:21,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981603 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981610 2024-11-26T09:56:21,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,616 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=205, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,616 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,616 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981610) roll requested 2024-11-26T09:56:21,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981616 2024-11-26T09:56:21,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981610 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981616 2024-11-26T09:56:21,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=206, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,625 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,625 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981616) roll requested 2024-11-26T09:56:21,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981625 2024-11-26T09:56:21,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981616 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981625 2024-11-26T09:56:21,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,640 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=207, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,641 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,641 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981625) roll requested 2024-11-26T09:56:21,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981641 2024-11-26T09:56:21,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981641 2024-11-26T09:56:21,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,647 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=208, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,647 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,647 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981641) roll requested 2024-11-26T09:56:21,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981647 2024-11-26T09:56:21,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981641 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981647 2024-11-26T09:56:21,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,654 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=209, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,654 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,654 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981647) roll requested 2024-11-26T09:56:21,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981654 2024-11-26T09:56:21,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981647 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981654 2024-11-26T09:56:21,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,661 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=210, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,661 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,661 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981654) roll requested 2024-11-26T09:56:21,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981662 2024-11-26T09:56:21,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981654 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981662 2024-11-26T09:56:21,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,673 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=211, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,673 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,673 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981662) roll requested 2024-11-26T09:56:21,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981673 2024-11-26T09:56:21,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981662 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981673 2024-11-26T09:56:21,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,684 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=212, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,684 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,684 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981673) roll requested 2024-11-26T09:56:21,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981685 2024-11-26T09:56:21,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981685 2024-11-26T09:56:21,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,691 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=213, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,691 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,691 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981685) roll requested 2024-11-26T09:56:21,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981692 2024-11-26T09:56:21,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981685 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981692 2024-11-26T09:56:21,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,702 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=214, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,702 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,702 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981692) roll requested 2024-11-26T09:56:21,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981702 2024-11-26T09:56:21,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981692 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981702 2024-11-26T09:56:21,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=215, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981702) roll requested 2024-11-26T09:56:21,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981709 2024-11-26T09:56:21,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981702 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981709 2024-11-26T09:56:21,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=216, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981709) roll requested 2024-11-26T09:56:21,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981716 2024-11-26T09:56:21,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981716 2024-11-26T09:56:21,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,726 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=217, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,726 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981716) roll requested 2024-11-26T09:56:21,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981726 2024-11-26T09:56:21,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981726 2024-11-26T09:56:21,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,732 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=218, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,732 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,732 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981726) roll requested 2024-11-26T09:56:21,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981732 2024-11-26T09:56:21,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981726 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981732 2024-11-26T09:56:21,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,744 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=219, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,744 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981732) roll requested 2024-11-26T09:56:21,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981744 2024-11-26T09:56:21,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981732 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981744 2024-11-26T09:56:21,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=220, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,753 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,753 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981744) roll requested 2024-11-26T09:56:21,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981753 2024-11-26T09:56:21,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981744 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981753 2024-11-26T09:56:21,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,772 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=221, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,772 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,772 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981753) roll requested 2024-11-26T09:56:21,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981772 2024-11-26T09:56:21,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981772 2024-11-26T09:56:21,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=222, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981772) roll requested 2024-11-26T09:56:21,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981788 2024-11-26T09:56:21,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981772 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981788 2024-11-26T09:56:21,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,812 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=223, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,812 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981788) roll requested 2024-11-26T09:56:21,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981812 2024-11-26T09:56:21,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981788 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981812 2024-11-26T09:56:21,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,825 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,825 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=224, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,825 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981812) roll requested 2024-11-26T09:56:21,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981825 2024-11-26T09:56:21,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981812 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981825 2024-11-26T09:56:21,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,836 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=225, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,836 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981825) roll requested 2024-11-26T09:56:21,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981836 2024-11-26T09:56:21,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981825 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981836 2024-11-26T09:56:21,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,848 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=226, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,848 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,848 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981836) roll requested 2024-11-26T09:56:21,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981848 2024-11-26T09:56:21,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981836 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981848 2024-11-26T09:56:21,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,872 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=227, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,872 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981848) roll requested 2024-11-26T09:56:21,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981872 2024-11-26T09:56:21,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981848 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981872 2024-11-26T09:56:21,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,888 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=228, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,888 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,888 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981872) roll requested 2024-11-26T09:56:21,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981888 2024-11-26T09:56:21,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981872 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981888 2024-11-26T09:56:21,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=229, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981888) roll requested 2024-11-26T09:56:21,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981907 2024-11-26T09:56:21,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981888 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981907 2024-11-26T09:56:21,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981888 is not closed yet, will try archiving it next time 2024-11-26T09:56:21,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=230, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,920 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981907) roll requested 2024-11-26T09:56:21,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981920 2024-11-26T09:56:21,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981907 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981920 2024-11-26T09:56:21,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,931 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=231, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,931 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,931 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981920) roll requested 2024-11-26T09:56:21,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981932 2024-11-26T09:56:21,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981932 2024-11-26T09:56:21,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=232, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,953 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981932) roll requested 2024-11-26T09:56:21,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981953 2024-11-26T09:56:21,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981932 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981953 2024-11-26T09:56:21,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,973 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=233, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,974 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981953) roll requested 2024-11-26T09:56:21,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981974 2024-11-26T09:56:21,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:21,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981974 2024-11-26T09:56:21,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:21,996 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=234, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:21,996 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,996 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:21,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:21,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981974) roll requested 2024-11-26T09:56:21,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614981997 2024-11-26T09:56:22,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981974 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981997 2024-11-26T09:56:22,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,020 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=235, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,020 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,020 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614981997) roll requested 2024-11-26T09:56:22,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982021 2024-11-26T09:56:22,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614981997 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982021 2024-11-26T09:56:22,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,046 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,046 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=236, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,046 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982021) roll requested 2024-11-26T09:56:22,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982046 2024-11-26T09:56:22,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982021 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982046 2024-11-26T09:56:22,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,075 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=237, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,076 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,079 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982046) roll requested 2024-11-26T09:56:22,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982079 2024-11-26T09:56:22,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982046 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982079 2024-11-26T09:56:22,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=238, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982079) roll requested 2024-11-26T09:56:22,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982104 2024-11-26T09:56:22,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982104 2024-11-26T09:56:22,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,127 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=239, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,127 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,127 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982104) roll requested 2024-11-26T09:56:22,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982128 2024-11-26T09:56:22,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982128 2024-11-26T09:56:22,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,156 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,157 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=240, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,157 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982128) roll requested 2024-11-26T09:56:22,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982157 2024-11-26T09:56:22,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982128 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982157 2024-11-26T09:56:22,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,180 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,180 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=241, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,181 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982157) roll requested 2024-11-26T09:56:22,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982181 2024-11-26T09:56:22,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982157 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982181 2024-11-26T09:56:22,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,199 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,199 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=242, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,200 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982181) roll requested 2024-11-26T09:56:22,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982200 2024-11-26T09:56:22,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982181 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982200 2024-11-26T09:56:22,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,219 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=243, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,219 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982200) roll requested 2024-11-26T09:56:22,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982219 2024-11-26T09:56:22,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982200 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982219 2024-11-26T09:56:22,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,234 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=244, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,234 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,234 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982219) roll requested 2024-11-26T09:56:22,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982234 2024-11-26T09:56:22,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982234 2024-11-26T09:56:22,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,245 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,245 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=245, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982234) roll requested 2024-11-26T09:56:22,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982245 2024-11-26T09:56:22,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982234 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982245 2024-11-26T09:56:22,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=246, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982245) roll requested 2024-11-26T09:56:22,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982268 2024-11-26T09:56:22,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982268 2024-11-26T09:56:22,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,275 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=247, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,275 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,275 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982268) roll requested 2024-11-26T09:56:22,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982276 2024-11-26T09:56:22,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982268 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982276 2024-11-26T09:56:22,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,282 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=248, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,282 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,282 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982276) roll requested 2024-11-26T09:56:22,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982282 2024-11-26T09:56:22,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982276 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982282 2024-11-26T09:56:22,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,289 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,289 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=249, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,289 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982282) roll requested 2024-11-26T09:56:22,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982289 2024-11-26T09:56:22,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982282 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982289 2024-11-26T09:56:22,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,295 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=250, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,295 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,295 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982289) roll requested 2024-11-26T09:56:22,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982295 2024-11-26T09:56:22,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982289 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982295 2024-11-26T09:56:22,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,302 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,302 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=251, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,302 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982295) roll requested 2024-11-26T09:56:22,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982302 2024-11-26T09:56:22,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982295 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982302 2024-11-26T09:56:22,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,308 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=252, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,309 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982302) roll requested 2024-11-26T09:56:22,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982309 2024-11-26T09:56:22,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982302 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982309 2024-11-26T09:56:22,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,315 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=253, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,315 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,315 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982309) roll requested 2024-11-26T09:56:22,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982315 2024-11-26T09:56:22,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982315 2024-11-26T09:56:22,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,322 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=254, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,322 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,322 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982315) roll requested 2024-11-26T09:56:22,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982322 2024-11-26T09:56:22,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982315 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982322 2024-11-26T09:56:22,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,331 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,331 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=255, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,331 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982322) roll requested 2024-11-26T09:56:22,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982332 2024-11-26T09:56:22,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982322 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982332 2024-11-26T09:56:22,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,343 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=256, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,343 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,343 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982332) roll requested 2024-11-26T09:56:22,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982344 2024-11-26T09:56:22,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982332 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982344 2024-11-26T09:56:22,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,352 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,352 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=257, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,352 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982344) roll requested 2024-11-26T09:56:22,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982352 2024-11-26T09:56:22,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982344 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982352 2024-11-26T09:56:22,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,359 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,359 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=258, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,359 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982352) roll requested 2024-11-26T09:56:22,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982359 2024-11-26T09:56:22,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982352 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982359 2024-11-26T09:56:22,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,367 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,367 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=259, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982359) roll requested 2024-11-26T09:56:22,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982368 2024-11-26T09:56:22,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982359 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982368 2024-11-26T09:56:22,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,376 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,376 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=260, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,376 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982368) roll requested 2024-11-26T09:56:22,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982376 2024-11-26T09:56:22,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982376 2024-11-26T09:56:22,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,383 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=261, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,383 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,383 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982376) roll requested 2024-11-26T09:56:22,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982383 2024-11-26T09:56:22,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982376 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982383 2024-11-26T09:56:22,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,393 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=262, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,393 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,393 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982383) roll requested 2024-11-26T09:56:22,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982393 2024-11-26T09:56:22,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982383 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982393 2024-11-26T09:56:22,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,404 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=263, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,404 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,404 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982393) roll requested 2024-11-26T09:56:22,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982405 2024-11-26T09:56:22,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982393 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982405 2024-11-26T09:56:22,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,417 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,417 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=264, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982405) roll requested 2024-11-26T09:56:22,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982417 2024-11-26T09:56:22,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982405 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982417 2024-11-26T09:56:22,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,425 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=265, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,425 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,425 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982417) roll requested 2024-11-26T09:56:22,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982425 2024-11-26T09:56:22,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982425 2024-11-26T09:56:22,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,434 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,434 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=266, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,434 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982425) roll requested 2024-11-26T09:56:22,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982434 2024-11-26T09:56:22,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982425 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982434 2024-11-26T09:56:22,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,448 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,448 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=267, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,448 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982434) roll requested 2024-11-26T09:56:22,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982448 2024-11-26T09:56:22,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982434 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982448 2024-11-26T09:56:22,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,466 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=268, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,466 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,466 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982448) roll requested 2024-11-26T09:56:22,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982467 2024-11-26T09:56:22,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982448 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982467 2024-11-26T09:56:22,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,476 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=269, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,476 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,476 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982467) roll requested 2024-11-26T09:56:22,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982476 2024-11-26T09:56:22,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982467 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982476 2024-11-26T09:56:22,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,486 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,486 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=270, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,486 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982476) roll requested 2024-11-26T09:56:22,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982486 2024-11-26T09:56:22,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982476 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982486 2024-11-26T09:56:22,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=271, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982486) roll requested 2024-11-26T09:56:22,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982504 2024-11-26T09:56:22,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982486 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982504 2024-11-26T09:56:22,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,520 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=272, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,520 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982504) roll requested 2024-11-26T09:56:22,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982520 2024-11-26T09:56:22,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982520 2024-11-26T09:56:22,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,532 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=273, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,532 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,532 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982520) roll requested 2024-11-26T09:56:22,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982532 2024-11-26T09:56:22,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982520 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982532 2024-11-26T09:56:22,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,548 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,548 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=274, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,548 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982532) roll requested 2024-11-26T09:56:22,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982548 2024-11-26T09:56:22,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982548 2024-11-26T09:56:22,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,560 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,560 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=275, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,560 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982548) roll requested 2024-11-26T09:56:22,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982560 2024-11-26T09:56:22,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982548 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982560 2024-11-26T09:56:22,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,572 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=276, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,572 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982560) roll requested 2024-11-26T09:56:22,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982572 2024-11-26T09:56:22,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982560 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982572 2024-11-26T09:56:22,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=277, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,589 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,590 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982572) roll requested 2024-11-26T09:56:22,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982590 2024-11-26T09:56:22,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982572 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982590 2024-11-26T09:56:22,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=278, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982590) roll requested 2024-11-26T09:56:22,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982608 2024-11-26T09:56:22,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982590 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982608 2024-11-26T09:56:22,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,621 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=279, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,622 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982608) roll requested 2024-11-26T09:56:22,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982622 2024-11-26T09:56:22,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982622 2024-11-26T09:56:22,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=280, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,652 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982622) roll requested 2024-11-26T09:56:22,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982653 2024-11-26T09:56:22,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982622 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982653 2024-11-26T09:56:22,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,673 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,673 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=281, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,673 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982653) roll requested 2024-11-26T09:56:22,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982673 2024-11-26T09:56:22,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982653 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982673 2024-11-26T09:56:22,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,692 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=282, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,692 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,692 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982673) roll requested 2024-11-26T09:56:22,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982693 2024-11-26T09:56:22,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982693 2024-11-26T09:56:22,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,713 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,714 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=283, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,714 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982693) roll requested 2024-11-26T09:56:22,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982714 2024-11-26T09:56:22,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982693 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982714 2024-11-26T09:56:22,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,728 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=284, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,728 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,729 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982714) roll requested 2024-11-26T09:56:22,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982729 2024-11-26T09:56:22,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982714 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982729 2024-11-26T09:56:22,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,744 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=285, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,745 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982729) roll requested 2024-11-26T09:56:22,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982745 2024-11-26T09:56:22,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982729 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982745 2024-11-26T09:56:22,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,759 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=286, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,759 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982745) roll requested 2024-11-26T09:56:22,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982759 2024-11-26T09:56:22,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982759 2024-11-26T09:56:22,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,772 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,772 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=287, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,772 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982759) roll requested 2024-11-26T09:56:22,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982772 2024-11-26T09:56:22,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982759 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982772 2024-11-26T09:56:22,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,781 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=288, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,781 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,781 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982772) roll requested 2024-11-26T09:56:22,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982781 2024-11-26T09:56:22,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982772 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982781 2024-11-26T09:56:22,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=289, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,789 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982781) roll requested 2024-11-26T09:56:22,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982789 2024-11-26T09:56:22,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982781 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982789 2024-11-26T09:56:22,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,795 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,795 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=290, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,795 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982789) roll requested 2024-11-26T09:56:22,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982795 2024-11-26T09:56:22,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982789 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982795 2024-11-26T09:56:22,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,802 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=291, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982795) roll requested 2024-11-26T09:56:22,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982802 2024-11-26T09:56:22,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982795 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982802 2024-11-26T09:56:22,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=292, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982802) roll requested 2024-11-26T09:56:22,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982808 2024-11-26T09:56:22,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982808 2024-11-26T09:56:22,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,814 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,815 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=293, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982808) roll requested 2024-11-26T09:56:22,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982815 2024-11-26T09:56:22,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982815 2024-11-26T09:56:22,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=294, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982815) roll requested 2024-11-26T09:56:22,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982821 2024-11-26T09:56:22,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982821 2024-11-26T09:56:22,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,827 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,827 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=295, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982821) roll requested 2024-11-26T09:56:22,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982828 2024-11-26T09:56:22,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982828 2024-11-26T09:56:22,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,834 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,834 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=296, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,834 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982828) roll requested 2024-11-26T09:56:22,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982834 2024-11-26T09:56:22,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982834 2024-11-26T09:56:22,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,841 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,841 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=297, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,841 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982834) roll requested 2024-11-26T09:56:22,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982841 2024-11-26T09:56:22,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982834 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982841 2024-11-26T09:56:22,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,847 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=298, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,847 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982841) roll requested 2024-11-26T09:56:22,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982847 2024-11-26T09:56:22,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982841 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982847 2024-11-26T09:56:22,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,853 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,853 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=299, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982847) roll requested 2024-11-26T09:56:22,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982854 2024-11-26T09:56:22,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982847 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982854 2024-11-26T09:56:22,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,860 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,860 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=300, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982854) roll requested 2024-11-26T09:56:22,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982860 2024-11-26T09:56:22,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982860 2024-11-26T09:56:22,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,868 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,868 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=301, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,868 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982860) roll requested 2024-11-26T09:56:22,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982869 2024-11-26T09:56:22,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982860 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982869 2024-11-26T09:56:22,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,875 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,875 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=302, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,875 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982869) roll requested 2024-11-26T09:56:22,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982875 2024-11-26T09:56:22,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982869 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982875 2024-11-26T09:56:22,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,884 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=303, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,885 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,885 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982875) roll requested 2024-11-26T09:56:22,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982885 2024-11-26T09:56:22,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982875 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982885 2024-11-26T09:56:22,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,892 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=304, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,892 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982885) roll requested 2024-11-26T09:56:22,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982892 2024-11-26T09:56:22,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982885 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982892 2024-11-26T09:56:22,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,899 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,899 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=305, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982892) roll requested 2024-11-26T09:56:22,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982900 2024-11-26T09:56:22,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982892 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982900 2024-11-26T09:56:22,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=306, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982900) roll requested 2024-11-26T09:56:22,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982906 2024-11-26T09:56:22,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982906 2024-11-26T09:56:22,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,913 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,913 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=307, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,913 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982906) roll requested 2024-11-26T09:56:22,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982913 2024-11-26T09:56:22,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982913 2024-11-26T09:56:22,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=308, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,920 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982913) roll requested 2024-11-26T09:56:22,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982920 2024-11-26T09:56:22,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982913 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982920 2024-11-26T09:56:22,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,928 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=309, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,928 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,928 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982920) roll requested 2024-11-26T09:56:22,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982928 2024-11-26T09:56:22,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982928 2024-11-26T09:56:22,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,937 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,937 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=310, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,937 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982928) roll requested 2024-11-26T09:56:22,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982937 2024-11-26T09:56:22,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982937 2024-11-26T09:56:22,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,948 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=311, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,948 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982937) roll requested 2024-11-26T09:56:22,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982948 2024-11-26T09:56:22,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982937 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982948 2024-11-26T09:56:22,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,955 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,955 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=312, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,955 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982948) roll requested 2024-11-26T09:56:22,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982955 2024-11-26T09:56:22,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982948 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982955 2024-11-26T09:56:22,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,961 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=313, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,961 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,961 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982955) roll requested 2024-11-26T09:56:22,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982961 2024-11-26T09:56:22,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982955 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982961 2024-11-26T09:56:22,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,968 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=314, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,968 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982961) roll requested 2024-11-26T09:56:22,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982968 2024-11-26T09:56:22,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982961 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982968 2024-11-26T09:56:22,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,974 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,974 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=315, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,974 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982968) roll requested 2024-11-26T09:56:22,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982974 2024-11-26T09:56:22,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982968 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982974 2024-11-26T09:56:22,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,980 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,980 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=316, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,981 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982974) roll requested 2024-11-26T09:56:22,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982981 2024-11-26T09:56:22,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982974 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982981 2024-11-26T09:56:22,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,987 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,987 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=317, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,987 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982981) roll requested 2024-11-26T09:56:22,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982987 2024-11-26T09:56:22,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:22,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982981 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982987 2024-11-26T09:56:22,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:22,994 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,994 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=318, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:22,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:22,994 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:22,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982987) roll requested 2024-11-26T09:56:22,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614982994 2024-11-26T09:56:23,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982994 2024-11-26T09:56:23,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,001 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,001 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=319, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,001 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614982994) roll requested 2024-11-26T09:56:23,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983001 2024-11-26T09:56:23,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614982994 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983001 2024-11-26T09:56:23,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,007 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=320, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983001) roll requested 2024-11-26T09:56:23,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983007 2024-11-26T09:56:23,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983001 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983007 2024-11-26T09:56:23,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,014 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=321, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983007) roll requested 2024-11-26T09:56:23,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983014 2024-11-26T09:56:23,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983014 2024-11-26T09:56:23,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,020 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,020 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=322, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,020 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983014) roll requested 2024-11-26T09:56:23,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983020 2024-11-26T09:56:23,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983020 2024-11-26T09:56:23,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,026 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,026 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=323, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,027 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983020) roll requested 2024-11-26T09:56:23,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983027 2024-11-26T09:56:23,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983020 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983027 2024-11-26T09:56:23,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,033 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,033 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=324, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,033 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983027) roll requested 2024-11-26T09:56:23,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983034 2024-11-26T09:56:23,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983027 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983034 2024-11-26T09:56:23,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,044 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=325, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,044 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983034) roll requested 2024-11-26T09:56:23,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983044 2024-11-26T09:56:23,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983034 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983044 2024-11-26T09:56:23,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,051 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=326, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,051 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,051 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983044) roll requested 2024-11-26T09:56:23,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983051 2024-11-26T09:56:23,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983044 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983051 2024-11-26T09:56:23,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,057 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,058 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=327, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,058 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983051) roll requested 2024-11-26T09:56:23,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983058 2024-11-26T09:56:23,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983051 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983058 2024-11-26T09:56:23,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,064 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,064 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=328, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,064 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983058) roll requested 2024-11-26T09:56:23,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983064 2024-11-26T09:56:23,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983058 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983064 2024-11-26T09:56:23,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,070 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=329, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,070 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983064) roll requested 2024-11-26T09:56:23,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983071 2024-11-26T09:56:23,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983064 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983071 2024-11-26T09:56:23,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,077 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=330, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,077 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983071) roll requested 2024-11-26T09:56:23,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983077 2024-11-26T09:56:23,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983071 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983077 2024-11-26T09:56:23,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,083 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,083 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=331, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,083 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983077) roll requested 2024-11-26T09:56:23,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983083 2024-11-26T09:56:23,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983077 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983083 2024-11-26T09:56:23,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,091 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=332, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,091 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,091 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983083) roll requested 2024-11-26T09:56:23,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983091 2024-11-26T09:56:23,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983083 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983091 2024-11-26T09:56:23,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=333, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983091) roll requested 2024-11-26T09:56:23,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983097 2024-11-26T09:56:23,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983091 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983097 2024-11-26T09:56:23,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=334, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983097) roll requested 2024-11-26T09:56:23,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983104 2024-11-26T09:56:23,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983104 2024-11-26T09:56:23,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,110 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=335, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,110 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,110 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983104) roll requested 2024-11-26T09:56:23,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983111 2024-11-26T09:56:23,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983111 2024-11-26T09:56:23,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,121 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,121 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=336, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983111) roll requested 2024-11-26T09:56:23,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983121 2024-11-26T09:56:23,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983111 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983121 2024-11-26T09:56:23,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,139 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,139 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=337, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,139 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983121) roll requested 2024-11-26T09:56:23,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983139 2024-11-26T09:56:23,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983139 2024-11-26T09:56:23,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,147 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,147 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=338, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,147 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983139) roll requested 2024-11-26T09:56:23,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983147 2024-11-26T09:56:23,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983139 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983147 2024-11-26T09:56:23,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,160 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,160 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=339, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,160 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983147) roll requested 2024-11-26T09:56:23,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983160 2024-11-26T09:56:23,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983147 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983160 2024-11-26T09:56:23,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,167 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,167 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=340, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,167 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983160) roll requested 2024-11-26T09:56:23,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983167 2024-11-26T09:56:23,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983160 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983167 2024-11-26T09:56:23,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=341, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,174 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983167) roll requested 2024-11-26T09:56:23,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983174 2024-11-26T09:56:23,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983167 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983174 2024-11-26T09:56:23,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,180 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,180 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=342, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,180 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983174) roll requested 2024-11-26T09:56:23,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983180 2024-11-26T09:56:23,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983174 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983180 2024-11-26T09:56:23,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,189 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,189 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=343, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983180) roll requested 2024-11-26T09:56:23,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983189 2024-11-26T09:56:23,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983189 2024-11-26T09:56:23,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=344, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983189) roll requested 2024-11-26T09:56:23,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983195 2024-11-26T09:56:23,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983195 2024-11-26T09:56:23,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,202 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=345, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,202 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,202 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983195) roll requested 2024-11-26T09:56:23,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983202 2024-11-26T09:56:23,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983195 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983202 2024-11-26T09:56:23,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,208 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,208 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=346, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,209 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983202) roll requested 2024-11-26T09:56:23,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983209 2024-11-26T09:56:23,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983202 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983209 2024-11-26T09:56:23,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,220 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=347, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,220 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,220 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983209) roll requested 2024-11-26T09:56:23,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983220 2024-11-26T09:56:23,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983220 2024-11-26T09:56:23,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,228 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,228 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=348, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,228 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983220) roll requested 2024-11-26T09:56:23,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983228 2024-11-26T09:56:23,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983220 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983228 2024-11-26T09:56:23,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,240 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=349, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,240 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983228) roll requested 2024-11-26T09:56:23,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983240 2024-11-26T09:56:23,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983228 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983240 2024-11-26T09:56:23,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,250 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,250 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=350, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,250 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983240) roll requested 2024-11-26T09:56:23,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983250 2024-11-26T09:56:23,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983240 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983250 2024-11-26T09:56:23,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,263 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,263 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=351, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,264 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983250) roll requested 2024-11-26T09:56:23,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983264 2024-11-26T09:56:23,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983250 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983264 2024-11-26T09:56:23,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,273 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,273 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=352, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,273 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983264) roll requested 2024-11-26T09:56:23,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983273 2024-11-26T09:56:23,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983273 2024-11-26T09:56:23,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,283 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,283 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=353, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,283 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983273) roll requested 2024-11-26T09:56:23,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983283 2024-11-26T09:56:23,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983273 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983283 2024-11-26T09:56:23,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,294 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,294 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=354, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,294 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983283) roll requested 2024-11-26T09:56:23,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983294 2024-11-26T09:56:23,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983283 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983294 2024-11-26T09:56:23,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,303 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=355, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,303 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983294) roll requested 2024-11-26T09:56:23,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983303 2024-11-26T09:56:23,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983294 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983303 2024-11-26T09:56:23,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,310 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=356, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,310 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983303) roll requested 2024-11-26T09:56:23,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983310 2024-11-26T09:56:23,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983303 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983310 2024-11-26T09:56:23,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,316 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=357, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,316 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983310) roll requested 2024-11-26T09:56:23,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983316 2024-11-26T09:56:23,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983310 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983316 2024-11-26T09:56:23,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,323 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,323 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=358, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,323 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983316) roll requested 2024-11-26T09:56:23,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983323 2024-11-26T09:56:23,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983316 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983323 2024-11-26T09:56:23,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,329 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=359, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983323) roll requested 2024-11-26T09:56:23,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983329 2024-11-26T09:56:23,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983323 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983329 2024-11-26T09:56:23,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,337 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=360, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,337 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,337 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983329) roll requested 2024-11-26T09:56:23,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983337 2024-11-26T09:56:23,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983337 2024-11-26T09:56:23,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,343 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,343 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=361, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,343 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983337) roll requested 2024-11-26T09:56:23,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983343 2024-11-26T09:56:23,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983337 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983343 2024-11-26T09:56:23,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,351 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,351 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=362, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,351 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983343) roll requested 2024-11-26T09:56:23,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983351 2024-11-26T09:56:23,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983343 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983351 2024-11-26T09:56:23,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,357 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=363, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,357 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,358 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983351) roll requested 2024-11-26T09:56:23,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983358 2024-11-26T09:56:23,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983351 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983358 2024-11-26T09:56:23,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=364, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983358) roll requested 2024-11-26T09:56:23,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983364 2024-11-26T09:56:23,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983358 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983364 2024-11-26T09:56:23,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,370 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,370 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=365, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,370 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983364) roll requested 2024-11-26T09:56:23,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983370 2024-11-26T09:56:23,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983370 2024-11-26T09:56:23,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,378 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,378 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=366, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,379 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983370) roll requested 2024-11-26T09:56:23,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983379 2024-11-26T09:56:23,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983379 2024-11-26T09:56:23,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,385 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,385 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=367, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983379) roll requested 2024-11-26T09:56:23,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983385 2024-11-26T09:56:23,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983379 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983385 2024-11-26T09:56:23,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,391 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,391 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=368, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,391 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983385) roll requested 2024-11-26T09:56:23,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983391 2024-11-26T09:56:23,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983391 2024-11-26T09:56:23,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=369, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,400 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983391) roll requested 2024-11-26T09:56:23,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983400 2024-11-26T09:56:23,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983391 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983400 2024-11-26T09:56:23,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,407 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,407 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=370, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,407 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983400) roll requested 2024-11-26T09:56:23,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983407 2024-11-26T09:56:23,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983400 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983407 2024-11-26T09:56:23,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,414 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=371, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,414 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983407) roll requested 2024-11-26T09:56:23,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983414 2024-11-26T09:56:23,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983407 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983414 2024-11-26T09:56:23,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=372, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983414) roll requested 2024-11-26T09:56:23,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983424 2024-11-26T09:56:23,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983414 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983424 2024-11-26T09:56:23,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,432 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=373, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,432 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983424) roll requested 2024-11-26T09:56:23,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983433 2024-11-26T09:56:23,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983433 2024-11-26T09:56:23,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,439 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,439 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=374, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,439 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983433) roll requested 2024-11-26T09:56:23,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983439 2024-11-26T09:56:23,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983439 2024-11-26T09:56:23,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,446 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,446 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=375, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,447 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983439) roll requested 2024-11-26T09:56:23,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983447 2024-11-26T09:56:23,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983439 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983447 2024-11-26T09:56:23,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,453 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,453 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=376, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,453 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983447) roll requested 2024-11-26T09:56:23,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983453 2024-11-26T09:56:23,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983447 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983453 2024-11-26T09:56:23,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,460 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=377, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,461 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983453) roll requested 2024-11-26T09:56:23,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983461 2024-11-26T09:56:23,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983453 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983461 2024-11-26T09:56:23,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,469 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,469 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=378, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,469 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983461) roll requested 2024-11-26T09:56:23,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983469 2024-11-26T09:56:23,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983469 2024-11-26T09:56:23,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,477 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=379, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983469) roll requested 2024-11-26T09:56:23,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983477 2024-11-26T09:56:23,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983469 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983477 2024-11-26T09:56:23,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=380, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983477) roll requested 2024-11-26T09:56:23,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983484 2024-11-26T09:56:23,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983484 2024-11-26T09:56:23,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,491 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,491 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=381, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,491 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983484) roll requested 2024-11-26T09:56:23,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983491 2024-11-26T09:56:23,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983484 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983491 2024-11-26T09:56:23,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,498 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=382, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,498 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983491) roll requested 2024-11-26T09:56:23,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983498 2024-11-26T09:56:23,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983491 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983498 2024-11-26T09:56:23,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,505 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,505 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=383, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,505 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983498) roll requested 2024-11-26T09:56:23,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983505 2024-11-26T09:56:23,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983498 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983505 2024-11-26T09:56:23,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=384, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,512 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983505) roll requested 2024-11-26T09:56:23,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983513 2024-11-26T09:56:23,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983505 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983513 2024-11-26T09:56:23,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,521 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,521 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=385, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,521 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983513) roll requested 2024-11-26T09:56:23,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983521 2024-11-26T09:56:23,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983513 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983521 2024-11-26T09:56:23,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,527 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,527 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=386, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,527 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983521) roll requested 2024-11-26T09:56:23,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983528 2024-11-26T09:56:23,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983528 2024-11-26T09:56:23,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,534 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,534 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=387, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,535 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983528) roll requested 2024-11-26T09:56:23,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983535 2024-11-26T09:56:23,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983528 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983535 2024-11-26T09:56:23,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,541 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,541 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=388, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,541 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983535) roll requested 2024-11-26T09:56:23,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983541 2024-11-26T09:56:23,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983535 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983541 2024-11-26T09:56:23,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,549 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=389, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,549 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983541) roll requested 2024-11-26T09:56:23,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983549 2024-11-26T09:56:23,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983541 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983549 2024-11-26T09:56:23,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,568 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=390, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,568 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983549) roll requested 2024-11-26T09:56:23,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983568 2024-11-26T09:56:23,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983549 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983568 2024-11-26T09:56:23,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,580 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=391, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,580 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,580 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983568) roll requested 2024-11-26T09:56:23,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983580 2024-11-26T09:56:23,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983568 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983580 2024-11-26T09:56:23,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,596 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=392, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,596 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,596 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983580) roll requested 2024-11-26T09:56:23,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983596 2024-11-26T09:56:23,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983580 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983596 2024-11-26T09:56:23,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,604 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=393, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,604 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983596) roll requested 2024-11-26T09:56:23,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983604 2024-11-26T09:56:23,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983596 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983604 2024-11-26T09:56:23,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,612 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,612 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=394, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,612 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983604) roll requested 2024-11-26T09:56:23,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983612 2024-11-26T09:56:23,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983604 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983612 2024-11-26T09:56:23,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=395, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983612) roll requested 2024-11-26T09:56:23,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983624 2024-11-26T09:56:23,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983612 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983624 2024-11-26T09:56:23,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,631 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,631 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=396, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,632 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983624) roll requested 2024-11-26T09:56:23,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983632 2024-11-26T09:56:23,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983624 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983632 2024-11-26T09:56:23,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,644 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=397, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,644 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983632) roll requested 2024-11-26T09:56:23,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983644 2024-11-26T09:56:23,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983632 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983644 2024-11-26T09:56:23,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,653 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=398, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,653 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,653 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983644) roll requested 2024-11-26T09:56:23,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983653 2024-11-26T09:56:23,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983644 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983653 2024-11-26T09:56:23,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,663 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,663 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=399, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,663 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983653) roll requested 2024-11-26T09:56:23,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983663 2024-11-26T09:56:23,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983653 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983663 2024-11-26T09:56:23,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,670 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,671 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=400, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,671 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983663) roll requested 2024-11-26T09:56:23,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983671 2024-11-26T09:56:23,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983663 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983671 2024-11-26T09:56:23,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,680 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=401, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,680 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983671) roll requested 2024-11-26T09:56:23,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983680 2024-11-26T09:56:23,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983671 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983680 2024-11-26T09:56:23,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=402, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,690 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983680) roll requested 2024-11-26T09:56:23,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983690 2024-11-26T09:56:23,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983680 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983690 2024-11-26T09:56:23,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,700 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=403, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,700 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983690) roll requested 2024-11-26T09:56:23,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983701 2024-11-26T09:56:23,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983690 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983701 2024-11-26T09:56:23,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=404, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983701) roll requested 2024-11-26T09:56:23,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983708 2024-11-26T09:56:23,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983708 2024-11-26T09:56:23,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=405, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983708) roll requested 2024-11-26T09:56:23,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983716 2024-11-26T09:56:23,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983716 2024-11-26T09:56:23,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,723 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,723 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=406, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983716) roll requested 2024-11-26T09:56:23,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983724 2024-11-26T09:56:23,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983724 2024-11-26T09:56:23,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,732 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=407, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,732 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,732 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983724) roll requested 2024-11-26T09:56:23,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983732 2024-11-26T09:56:23,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983732 2024-11-26T09:56:23,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=408, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983732) roll requested 2024-11-26T09:56:23,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983741 2024-11-26T09:56:23,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983732 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983741 2024-11-26T09:56:23,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=409, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,748 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983741) roll requested 2024-11-26T09:56:23,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983748 2024-11-26T09:56:23,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983748 2024-11-26T09:56:23,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=410, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983748) roll requested 2024-11-26T09:56:23,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983756 2024-11-26T09:56:23,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983748 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983756 2024-11-26T09:56:23,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,768 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,768 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=411, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983756) roll requested 2024-11-26T09:56:23,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983768 2024-11-26T09:56:23,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983768 2024-11-26T09:56:23,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=412, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,776 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,776 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983768) roll requested 2024-11-26T09:56:23,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983776 2024-11-26T09:56:23,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983768 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983776 2024-11-26T09:56:23,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=413, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983776) roll requested 2024-11-26T09:56:23,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983788 2024-11-26T09:56:23,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983776 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983788 2024-11-26T09:56:23,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,800 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=414, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,800 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983788) roll requested 2024-11-26T09:56:23,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983800 2024-11-26T09:56:23,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983788 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983800 2024-11-26T09:56:23,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=415, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983800) roll requested 2024-11-26T09:56:23,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983808 2024-11-26T09:56:23,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983800 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983808 2024-11-26T09:56:23,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,816 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,816 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=416, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,816 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983808) roll requested 2024-11-26T09:56:23,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983816 2024-11-26T09:56:23,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983816 2024-11-26T09:56:23,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,824 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=417, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,824 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983816) roll requested 2024-11-26T09:56:23,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983824 2024-11-26T09:56:23,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983816 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983824 2024-11-26T09:56:23,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,832 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,832 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=418, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,832 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983824) roll requested 2024-11-26T09:56:23,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983832 2024-11-26T09:56:23,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983824 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983832 2024-11-26T09:56:23,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,841 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,841 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=419, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,841 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983832) roll requested 2024-11-26T09:56:23,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983841 2024-11-26T09:56:23,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983832 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983841 2024-11-26T09:56:23,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,848 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,848 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=420, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,848 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983841) roll requested 2024-11-26T09:56:23,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983848 2024-11-26T09:56:23,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983841 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983848 2024-11-26T09:56:23,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,857 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,858 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=421, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,858 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983848) roll requested 2024-11-26T09:56:23,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983858 2024-11-26T09:56:23,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983848 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983858 2024-11-26T09:56:23,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,867 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,867 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=422, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,867 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983858) roll requested 2024-11-26T09:56:23,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983867 2024-11-26T09:56:23,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983867 2024-11-26T09:56:23,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,877 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,877 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=423, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,877 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983867) roll requested 2024-11-26T09:56:23,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983877 2024-11-26T09:56:23,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983867 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983877 2024-11-26T09:56:23,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=424, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,887 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983877) roll requested 2024-11-26T09:56:23,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983887 2024-11-26T09:56:23,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983877 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983887 2024-11-26T09:56:23,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,897 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,897 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=425, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,897 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983887) roll requested 2024-11-26T09:56:23,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983897 2024-11-26T09:56:23,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983887 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983897 2024-11-26T09:56:23,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,905 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,905 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=426, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,905 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983897) roll requested 2024-11-26T09:56:23,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983905 2024-11-26T09:56:23,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983905 2024-11-26T09:56:23,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983897 is not closed yet, will try archiving it next time 2024-11-26T09:56:23,912 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,912 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=427, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,912 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983905) roll requested 2024-11-26T09:56:23,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983912 2024-11-26T09:56:23,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983905 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983912 2024-11-26T09:56:23,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,921 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=428, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983912) roll requested 2024-11-26T09:56:23,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983921 2024-11-26T09:56:23,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983912 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983921 2024-11-26T09:56:23,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,930 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,930 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=429, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,931 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983921) roll requested 2024-11-26T09:56:23,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983931 2024-11-26T09:56:23,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983931 2024-11-26T09:56:23,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,944 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,944 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=430, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,944 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983931) roll requested 2024-11-26T09:56:23,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983944 2024-11-26T09:56:23,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983931 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983944 2024-11-26T09:56:23,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,951 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,951 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=431, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,951 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983944) roll requested 2024-11-26T09:56:23,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983951 2024-11-26T09:56:23,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983944 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983951 2024-11-26T09:56:23,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=432, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983951) roll requested 2024-11-26T09:56:23,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983960 2024-11-26T09:56:23,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983951 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983960 2024-11-26T09:56:23,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=433, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983960) roll requested 2024-11-26T09:56:23,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983972 2024-11-26T09:56:23,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983972 2024-11-26T09:56:23,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,979 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=434, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983972) roll requested 2024-11-26T09:56:23,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983979 2024-11-26T09:56:23,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983979 2024-11-26T09:56:23,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=435, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983979) roll requested 2024-11-26T09:56:23,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983986 2024-11-26T09:56:23,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:23,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983986 2024-11-26T09:56:23,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:23,996 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,997 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=436, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:23,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:23,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:23,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983986) roll requested 2024-11-26T09:56:23,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614983997 2024-11-26T09:56:24,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983986 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983997 2024-11-26T09:56:24,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,004 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=437, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,005 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614983997) roll requested 2024-11-26T09:56:24,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984005 2024-11-26T09:56:24,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614983997 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984005 2024-11-26T09:56:24,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,011 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=438, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,011 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984005) roll requested 2024-11-26T09:56:24,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984011 2024-11-26T09:56:24,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984011 2024-11-26T09:56:24,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=439, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,018 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984011) roll requested 2024-11-26T09:56:24,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984018 2024-11-26T09:56:24,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984011 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984018 2024-11-26T09:56:24,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=440, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,026 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984018) roll requested 2024-11-26T09:56:24,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984026 2024-11-26T09:56:24,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984018 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984026 2024-11-26T09:56:24,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,032 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,032 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=441, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,032 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984026) roll requested 2024-11-26T09:56:24,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984033 2024-11-26T09:56:24,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984026 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984033 2024-11-26T09:56:24,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,041 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,042 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=442, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,042 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984033) roll requested 2024-11-26T09:56:24,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984042 2024-11-26T09:56:24,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984033 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984042 2024-11-26T09:56:24,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,052 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=443, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,052 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,052 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984042) roll requested 2024-11-26T09:56:24,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984052 2024-11-26T09:56:24,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984042 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984052 2024-11-26T09:56:24,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,060 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=444, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,060 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984052) roll requested 2024-11-26T09:56:24,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984060 2024-11-26T09:56:24,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984052 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984060 2024-11-26T09:56:24,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,068 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,068 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=445, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,068 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984060) roll requested 2024-11-26T09:56:24,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984068 2024-11-26T09:56:24,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984060 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984068 2024-11-26T09:56:24,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,076 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,076 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=446, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,076 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984068) roll requested 2024-11-26T09:56:24,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984076 2024-11-26T09:56:24,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984068 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984076 2024-11-26T09:56:24,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,085 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=447, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,085 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984076) roll requested 2024-11-26T09:56:24,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984085 2024-11-26T09:56:24,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984076 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984085 2024-11-26T09:56:24,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,091 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=448, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,091 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,091 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984085) roll requested 2024-11-26T09:56:24,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984092 2024-11-26T09:56:24,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984092 2024-11-26T09:56:24,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,098 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,098 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=449, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984092) roll requested 2024-11-26T09:56:24,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984098 2024-11-26T09:56:24,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984092 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984098 2024-11-26T09:56:24,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,106 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,106 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=450, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,106 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984098) roll requested 2024-11-26T09:56:24,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984106 2024-11-26T09:56:24,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984098 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984106 2024-11-26T09:56:24,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,113 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,113 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=451, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984106) roll requested 2024-11-26T09:56:24,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984113 2024-11-26T09:56:24,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984106 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984113 2024-11-26T09:56:24,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,120 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=452, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,120 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984113) roll requested 2024-11-26T09:56:24,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984120 2024-11-26T09:56:24,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984120 2024-11-26T09:56:24,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,126 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=453, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984120) roll requested 2024-11-26T09:56:24,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984126 2024-11-26T09:56:24,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984120 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984126 2024-11-26T09:56:24,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,134 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,134 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=454, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,134 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984126) roll requested 2024-11-26T09:56:24,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984134 2024-11-26T09:56:24,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984126 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984134 2024-11-26T09:56:24,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,146 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=455, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,146 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,146 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984134) roll requested 2024-11-26T09:56:24,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984146 2024-11-26T09:56:24,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984134 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984146 2024-11-26T09:56:24,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,152 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=456, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,152 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,152 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984146) roll requested 2024-11-26T09:56:24,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984152 2024-11-26T09:56:24,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984146 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984152 2024-11-26T09:56:24,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,159 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=457, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,159 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984152) roll requested 2024-11-26T09:56:24,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984159 2024-11-26T09:56:24,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984152 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984159 2024-11-26T09:56:24,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,172 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=458, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,172 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984159) roll requested 2024-11-26T09:56:24,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984172 2024-11-26T09:56:24,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984159 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984172 2024-11-26T09:56:24,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=459, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,185 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984172) roll requested 2024-11-26T09:56:24,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984185 2024-11-26T09:56:24,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984172 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984185 2024-11-26T09:56:24,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,194 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,194 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=460, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,194 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984185) roll requested 2024-11-26T09:56:24,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984194 2024-11-26T09:56:24,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984194 2024-11-26T09:56:24,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,207 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=461, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,207 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,207 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984194) roll requested 2024-11-26T09:56:24,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984207 2024-11-26T09:56:24,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984194 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984207 2024-11-26T09:56:24,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,214 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,214 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=462, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,214 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984207) roll requested 2024-11-26T09:56:24,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984214 2024-11-26T09:56:24,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984207 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984214 2024-11-26T09:56:24,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,222 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,222 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=463, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,222 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984214) roll requested 2024-11-26T09:56:24,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984222 2024-11-26T09:56:24,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984214 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984222 2024-11-26T09:56:24,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,231 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=464, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,231 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,231 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984222) roll requested 2024-11-26T09:56:24,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984231 2024-11-26T09:56:24,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984222 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984231 2024-11-26T09:56:24,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,239 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=465, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,239 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984231) roll requested 2024-11-26T09:56:24,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984239 2024-11-26T09:56:24,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984231 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984239 2024-11-26T09:56:24,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,245 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,246 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=466, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,246 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984239) roll requested 2024-11-26T09:56:24,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984246 2024-11-26T09:56:24,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984239 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984246 2024-11-26T09:56:24,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,257 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,257 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=467, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984246) roll requested 2024-11-26T09:56:24,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984257 2024-11-26T09:56:24,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984246 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984257 2024-11-26T09:56:24,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,269 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,269 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=468, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984257) roll requested 2024-11-26T09:56:24,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984269 2024-11-26T09:56:24,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984269 2024-11-26T09:56:24,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,276 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=469, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984269) roll requested 2024-11-26T09:56:24,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984277 2024-11-26T09:56:24,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984277 2024-11-26T09:56:24,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=470, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984277) roll requested 2024-11-26T09:56:24,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984285 2024-11-26T09:56:24,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984285 2024-11-26T09:56:24,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,294 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=471, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,294 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,294 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984285) roll requested 2024-11-26T09:56:24,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984294 2024-11-26T09:56:24,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984294 2024-11-26T09:56:24,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,304 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=472, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,304 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984294) roll requested 2024-11-26T09:56:24,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984304 2024-11-26T09:56:24,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984294 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984304 2024-11-26T09:56:24,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,311 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,311 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=473, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,311 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984304) roll requested 2024-11-26T09:56:24,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984311 2024-11-26T09:56:24,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984304 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984311 2024-11-26T09:56:24,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,318 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,318 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=474, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,318 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984311) roll requested 2024-11-26T09:56:24,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984318 2024-11-26T09:56:24,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984311 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984318 2024-11-26T09:56:24,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,326 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,326 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=475, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,327 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984318) roll requested 2024-11-26T09:56:24,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984327 2024-11-26T09:56:24,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984318 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984327 2024-11-26T09:56:24,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,335 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,335 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=476, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,335 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984327) roll requested 2024-11-26T09:56:24,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984335 2024-11-26T09:56:24,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984327 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984335 2024-11-26T09:56:24,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,342 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,342 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=477, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984335) roll requested 2024-11-26T09:56:24,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984342 2024-11-26T09:56:24,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984335 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984342 2024-11-26T09:56:24,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=478, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984342) roll requested 2024-11-26T09:56:24,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984349 2024-11-26T09:56:24,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984342 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984349 2024-11-26T09:56:24,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=479, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984349) roll requested 2024-11-26T09:56:24,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984355 2024-11-26T09:56:24,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984355 2024-11-26T09:56:24,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,361 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,361 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=480, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984355) roll requested 2024-11-26T09:56:24,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984361 2024-11-26T09:56:24,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984361 2024-11-26T09:56:24,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=481, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984361) roll requested 2024-11-26T09:56:24,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984368 2024-11-26T09:56:24,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984368 2024-11-26T09:56:24,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,374 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,374 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=482, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,374 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984368) roll requested 2024-11-26T09:56:24,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984374 2024-11-26T09:56:24,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984374 2024-11-26T09:56:24,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=483, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984374) roll requested 2024-11-26T09:56:24,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984381 2024-11-26T09:56:24,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984374 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984381 2024-11-26T09:56:24,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,387 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,387 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=484, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,387 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984381) roll requested 2024-11-26T09:56:24,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984387 2024-11-26T09:56:24,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984387 2024-11-26T09:56:24,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,393 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,393 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=485, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,393 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984387) roll requested 2024-11-26T09:56:24,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984394 2024-11-26T09:56:24,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984387 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984394 2024-11-26T09:56:24,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,399 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=486, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,400 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984394) roll requested 2024-11-26T09:56:24,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984400 2024-11-26T09:56:24,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984394 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984400 2024-11-26T09:56:24,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,406 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,406 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=487, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,407 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984400) roll requested 2024-11-26T09:56:24,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984407 2024-11-26T09:56:24,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984400 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984407 2024-11-26T09:56:24,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,417 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,417 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=488, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984407) roll requested 2024-11-26T09:56:24,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984417 2024-11-26T09:56:24,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984407 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984417 2024-11-26T09:56:24,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=489, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984417) roll requested 2024-11-26T09:56:24,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984424 2024-11-26T09:56:24,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984424 2024-11-26T09:56:24,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,430 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,430 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=490, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,430 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984424) roll requested 2024-11-26T09:56:24,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984430 2024-11-26T09:56:24,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984430 2024-11-26T09:56:24,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,440 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,440 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=491, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,440 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984430) roll requested 2024-11-26T09:56:24,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984440 2024-11-26T09:56:24,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984430 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984440 2024-11-26T09:56:24,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,447 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,447 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=492, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,447 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984440) roll requested 2024-11-26T09:56:24,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984447 2024-11-26T09:56:24,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984447 2024-11-26T09:56:24,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,454 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,454 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=493, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,454 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984447) roll requested 2024-11-26T09:56:24,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984454 2024-11-26T09:56:24,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984447 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984454 2024-11-26T09:56:24,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,460 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=494, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,460 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984454) roll requested 2024-11-26T09:56:24,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984460 2024-11-26T09:56:24,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984454 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984460 2024-11-26T09:56:24,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,467 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=495, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,467 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984460) roll requested 2024-11-26T09:56:24,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984467 2024-11-26T09:56:24,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984460 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984467 2024-11-26T09:56:24,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,474 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=496, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,474 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984467) roll requested 2024-11-26T09:56:24,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984474 2024-11-26T09:56:24,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984467 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984474 2024-11-26T09:56:24,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,480 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,480 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=497, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,480 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984474) roll requested 2024-11-26T09:56:24,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984480 2024-11-26T09:56:24,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984474 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984480 2024-11-26T09:56:24,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,486 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,486 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=498, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,486 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984480) roll requested 2024-11-26T09:56:24,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984487 2024-11-26T09:56:24,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984480 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984487 2024-11-26T09:56:24,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,493 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=499, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984487) roll requested 2024-11-26T09:56:24,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984493 2024-11-26T09:56:24,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984487 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984493 2024-11-26T09:56:24,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,499 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,499 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=500, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,499 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984493) roll requested 2024-11-26T09:56:24,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984499 2024-11-26T09:56:24,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984499 2024-11-26T09:56:24,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,505 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,506 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=501, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,506 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984499) roll requested 2024-11-26T09:56:24,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984506 2024-11-26T09:56:24,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984499 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984506 2024-11-26T09:56:24,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=502, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,512 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984506) roll requested 2024-11-26T09:56:24,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984512 2024-11-26T09:56:24,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984506 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984512 2024-11-26T09:56:24,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=503, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,519 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984512) roll requested 2024-11-26T09:56:24,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984519 2024-11-26T09:56:24,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984512 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984519 2024-11-26T09:56:24,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,525 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,525 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=504, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,525 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984519) roll requested 2024-11-26T09:56:24,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984525 2024-11-26T09:56:24,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984519 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984525 2024-11-26T09:56:24,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,532 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=505, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,532 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984525) roll requested 2024-11-26T09:56:24,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984532 2024-11-26T09:56:24,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984525 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984532 2024-11-26T09:56:24,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,538 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,538 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=506, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,539 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984532) roll requested 2024-11-26T09:56:24,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984539 2024-11-26T09:56:24,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984539 2024-11-26T09:56:24,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,545 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,545 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=507, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,545 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984539) roll requested 2024-11-26T09:56:24,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984545 2024-11-26T09:56:24,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984539 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984545 2024-11-26T09:56:24,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,551 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,551 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=508, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,551 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984545) roll requested 2024-11-26T09:56:24,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984551 2024-11-26T09:56:24,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984545 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984551 2024-11-26T09:56:24,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,558 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,558 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=509, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,558 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984551) roll requested 2024-11-26T09:56:24,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984558 2024-11-26T09:56:24,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984551 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984558 2024-11-26T09:56:24,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=510, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,564 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984558) roll requested 2024-11-26T09:56:24,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984564 2024-11-26T09:56:24,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984558 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984564 2024-11-26T09:56:24,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,571 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,571 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=511, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,571 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984564) roll requested 2024-11-26T09:56:24,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984571 2024-11-26T09:56:24,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984571 2024-11-26T09:56:24,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,578 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,578 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=512, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,578 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984571) roll requested 2024-11-26T09:56:24,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984578 2024-11-26T09:56:24,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984571 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984578 2024-11-26T09:56:24,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,584 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,584 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=513, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,584 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984578) roll requested 2024-11-26T09:56:24,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984585 2024-11-26T09:56:24,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984578 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984585 2024-11-26T09:56:24,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,591 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,591 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=514, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,591 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984585) roll requested 2024-11-26T09:56:24,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984591 2024-11-26T09:56:24,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984591 2024-11-26T09:56:24,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,597 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,597 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=515, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,597 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984591) roll requested 2024-11-26T09:56:24,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984598 2024-11-26T09:56:24,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984591 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984598 2024-11-26T09:56:24,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,604 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=516, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,604 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984598) roll requested 2024-11-26T09:56:24,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984604 2024-11-26T09:56:24,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984598 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984604 2024-11-26T09:56:24,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,610 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,610 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=517, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,610 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984604) roll requested 2024-11-26T09:56:24,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984610 2024-11-26T09:56:24,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984604 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984610 2024-11-26T09:56:24,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,618 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,618 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=518, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,618 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984610) roll requested 2024-11-26T09:56:24,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984618 2024-11-26T09:56:24,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984610 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984618 2024-11-26T09:56:24,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=519, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984618) roll requested 2024-11-26T09:56:24,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984624 2024-11-26T09:56:24,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984618 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984624 2024-11-26T09:56:24,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,630 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,631 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=520, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,631 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984624) roll requested 2024-11-26T09:56:24,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984631 2024-11-26T09:56:24,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984624 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984631 2024-11-26T09:56:24,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,637 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=521, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,638 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984631) roll requested 2024-11-26T09:56:24,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984638 2024-11-26T09:56:24,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984631 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984638 2024-11-26T09:56:24,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,644 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=522, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,644 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984638) roll requested 2024-11-26T09:56:24,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984644 2024-11-26T09:56:24,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984638 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984644 2024-11-26T09:56:24,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,650 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=523, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984644) roll requested 2024-11-26T09:56:24,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984651 2024-11-26T09:56:24,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984644 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984651 2024-11-26T09:56:24,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,657 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,657 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=524, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,657 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984651) roll requested 2024-11-26T09:56:24,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984657 2024-11-26T09:56:24,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984657 2024-11-26T09:56:24,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,663 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,663 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=525, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,663 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984657) roll requested 2024-11-26T09:56:24,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984663 2024-11-26T09:56:24,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984657 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984663 2024-11-26T09:56:24,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,669 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,669 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=526, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,670 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984663) roll requested 2024-11-26T09:56:24,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984670 2024-11-26T09:56:24,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984663 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984670 2024-11-26T09:56:24,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,675 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,676 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=527, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,676 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984670) roll requested 2024-11-26T09:56:24,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984676 2024-11-26T09:56:24,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984670 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984676 2024-11-26T09:56:24,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,682 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,682 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=528, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,682 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984676) roll requested 2024-11-26T09:56:24,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984682 2024-11-26T09:56:24,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984676 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984682 2024-11-26T09:56:24,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=529, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984682) roll requested 2024-11-26T09:56:24,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984689 2024-11-26T09:56:24,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984682 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984689 2024-11-26T09:56:24,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,695 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=530, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,695 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984689) roll requested 2024-11-26T09:56:24,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984696 2024-11-26T09:56:24,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984696 2024-11-26T09:56:24,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,702 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,702 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=531, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,702 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984696) roll requested 2024-11-26T09:56:24,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984702 2024-11-26T09:56:24,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984702 2024-11-26T09:56:24,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=532, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984702) roll requested 2024-11-26T09:56:24,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984709 2024-11-26T09:56:24,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984702 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984709 2024-11-26T09:56:24,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,715 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=533, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,715 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984709) roll requested 2024-11-26T09:56:24,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984715 2024-11-26T09:56:24,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984715 2024-11-26T09:56:24,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,721 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,721 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=534, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,721 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984715) roll requested 2024-11-26T09:56:24,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984721 2024-11-26T09:56:24,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984715 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984721 2024-11-26T09:56:24,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,727 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,728 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=535, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,728 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984721) roll requested 2024-11-26T09:56:24,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984728 2024-11-26T09:56:24,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984721 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984728 2024-11-26T09:56:24,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=536, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984728) roll requested 2024-11-26T09:56:24,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984734 2024-11-26T09:56:24,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984728 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984734 2024-11-26T09:56:24,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,741 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,741 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=537, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,741 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984734) roll requested 2024-11-26T09:56:24,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984741 2024-11-26T09:56:24,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984734 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984741 2024-11-26T09:56:24,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,749 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,749 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=538, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984741) roll requested 2024-11-26T09:56:24,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984749 2024-11-26T09:56:24,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984749 2024-11-26T09:56:24,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=539, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984749) roll requested 2024-11-26T09:56:24,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984756 2024-11-26T09:56:24,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984756 2024-11-26T09:56:24,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,765 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,765 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=540, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,765 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984756) roll requested 2024-11-26T09:56:24,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984765 2024-11-26T09:56:24,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984765 2024-11-26T09:56:24,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,772 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=541, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984765) roll requested 2024-11-26T09:56:24,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984773 2024-11-26T09:56:24,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984765 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984773 2024-11-26T09:56:24,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,780 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,780 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=542, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,780 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984773) roll requested 2024-11-26T09:56:24,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984780 2024-11-26T09:56:24,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984780 2024-11-26T09:56:24,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=543, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984780) roll requested 2024-11-26T09:56:24,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984788 2024-11-26T09:56:24,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984780 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984788 2024-11-26T09:56:24,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,797 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,797 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=544, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,797 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984788) roll requested 2024-11-26T09:56:24,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984798 2024-11-26T09:56:24,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984788 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984798 2024-11-26T09:56:24,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,804 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,805 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=545, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,805 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984798) roll requested 2024-11-26T09:56:24,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984805 2024-11-26T09:56:24,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984798 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984805 2024-11-26T09:56:24,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,815 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,815 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=546, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984805) roll requested 2024-11-26T09:56:24,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984815 2024-11-26T09:56:24,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984805 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984815 2024-11-26T09:56:24,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,823 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,823 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=547, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,823 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984815) roll requested 2024-11-26T09:56:24,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984823 2024-11-26T09:56:24,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984823 2024-11-26T09:56:24,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,835 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,835 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=548, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,835 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984823) roll requested 2024-11-26T09:56:24,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984835 2024-11-26T09:56:24,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984835 2024-11-26T09:56:24,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,842 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,843 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=549, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,843 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984835) roll requested 2024-11-26T09:56:24,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984843 2024-11-26T09:56:24,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984835 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984843 2024-11-26T09:56:24,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,849 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,849 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=550, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,849 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984843) roll requested 2024-11-26T09:56:24,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984849 2024-11-26T09:56:24,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984843 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984849 2024-11-26T09:56:24,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=551, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984849) roll requested 2024-11-26T09:56:24,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984856 2024-11-26T09:56:24,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984856 2024-11-26T09:56:24,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,862 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,862 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=552, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,862 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984856) roll requested 2024-11-26T09:56:24,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984862 2024-11-26T09:56:24,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984862 2024-11-26T09:56:24,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,869 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,869 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=553, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,869 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984862) roll requested 2024-11-26T09:56:24,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984869 2024-11-26T09:56:24,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984862 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984869 2024-11-26T09:56:24,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,875 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,875 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=554, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,875 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984869) roll requested 2024-11-26T09:56:24,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984875 2024-11-26T09:56:24,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984869 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984875 2024-11-26T09:56:24,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,881 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,881 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=555, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,881 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984875) roll requested 2024-11-26T09:56:24,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984881 2024-11-26T09:56:24,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984875 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984881 2024-11-26T09:56:24,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,888 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=556, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,888 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984881) roll requested 2024-11-26T09:56:24,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984888 2024-11-26T09:56:24,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984881 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984888 2024-11-26T09:56:24,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,894 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,894 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=557, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,894 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984888) roll requested 2024-11-26T09:56:24,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984894 2024-11-26T09:56:24,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984888 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984894 2024-11-26T09:56:24,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=558, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984894) roll requested 2024-11-26T09:56:24,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984900 2024-11-26T09:56:24,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984900 2024-11-26T09:56:24,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=559, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984900) roll requested 2024-11-26T09:56:24,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984906 2024-11-26T09:56:24,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984906 2024-11-26T09:56:24,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,913 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,913 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=560, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,913 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984906) roll requested 2024-11-26T09:56:24,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984913 2024-11-26T09:56:24,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984913 2024-11-26T09:56:24,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=561, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,920 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984913) roll requested 2024-11-26T09:56:24,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984920 2024-11-26T09:56:24,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984913 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984920 2024-11-26T09:56:24,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,928 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,928 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=562, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,928 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984920) roll requested 2024-11-26T09:56:24,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984928 2024-11-26T09:56:24,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984928 2024-11-26T09:56:24,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,934 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,934 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=563, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,934 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984928) roll requested 2024-11-26T09:56:24,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984935 2024-11-26T09:56:24,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984935 2024-11-26T09:56:24,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,941 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,941 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=564, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,941 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984935) roll requested 2024-11-26T09:56:24,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984941 2024-11-26T09:56:24,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984935 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984941 2024-11-26T09:56:24,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,947 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,947 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=565, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,948 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984941) roll requested 2024-11-26T09:56:24,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984948 2024-11-26T09:56:24,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984941 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984948 2024-11-26T09:56:24,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,954 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,954 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=566, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,954 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984948) roll requested 2024-11-26T09:56:24,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984954 2024-11-26T09:56:24,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984948 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984954 2024-11-26T09:56:24,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=567, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,961 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984954) roll requested 2024-11-26T09:56:24,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984961 2024-11-26T09:56:24,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984954 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984961 2024-11-26T09:56:24,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,967 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=568, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,967 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984961) roll requested 2024-11-26T09:56:24,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984967 2024-11-26T09:56:24,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984961 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984967 2024-11-26T09:56:24,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,973 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,974 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=569, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,974 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984967) roll requested 2024-11-26T09:56:24,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984974 2024-11-26T09:56:24,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984967 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984974 2024-11-26T09:56:24,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,980 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,980 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=570, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,980 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984974) roll requested 2024-11-26T09:56:24,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984980 2024-11-26T09:56:24,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984974 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984980 2024-11-26T09:56:24,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,986 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=571, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,987 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984980) roll requested 2024-11-26T09:56:24,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984987 2024-11-26T09:56:24,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984980 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984987 2024-11-26T09:56:24,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,993 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=572, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984987) roll requested 2024-11-26T09:56:24,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984993 2024-11-26T09:56:24,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:24,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984993 2024-11-26T09:56:24,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:24,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:24,999 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=573, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:24,999 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:24,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984993) roll requested 2024-11-26T09:56:24,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614984999 2024-11-26T09:56:25,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984999 2024-11-26T09:56:25,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,005 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,006 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=574, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,006 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614984999) roll requested 2024-11-26T09:56:25,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985006 2024-11-26T09:56:25,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614984999 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985006 2024-11-26T09:56:25,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,012 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,012 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=575, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,012 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985006) roll requested 2024-11-26T09:56:25,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985012 2024-11-26T09:56:25,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985006 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985012 2024-11-26T09:56:25,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=576, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,018 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985012) roll requested 2024-11-26T09:56:25,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985018 2024-11-26T09:56:25,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985012 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985018 2024-11-26T09:56:25,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,024 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=577, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985018) roll requested 2024-11-26T09:56:25,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985025 2024-11-26T09:56:25,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985018 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985025 2024-11-26T09:56:25,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,031 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,031 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=578, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,031 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985025) roll requested 2024-11-26T09:56:25,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985031 2024-11-26T09:56:25,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985031 2024-11-26T09:56:25,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,038 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=579, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,038 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985031) roll requested 2024-11-26T09:56:25,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985038 2024-11-26T09:56:25,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985031 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985038 2024-11-26T09:56:25,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,044 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=580, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,044 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985038) roll requested 2024-11-26T09:56:25,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985044 2024-11-26T09:56:25,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985038 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985044 2024-11-26T09:56:25,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,050 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,050 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=581, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,050 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985044) roll requested 2024-11-26T09:56:25,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985050 2024-11-26T09:56:25,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985044 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985050 2024-11-26T09:56:25,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,056 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,057 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=582, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,057 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985050) roll requested 2024-11-26T09:56:25,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985057 2024-11-26T09:56:25,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985050 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985057 2024-11-26T09:56:25,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,063 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,063 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=583, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,063 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985057) roll requested 2024-11-26T09:56:25,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985063 2024-11-26T09:56:25,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985057 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985063 2024-11-26T09:56:25,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,071 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=584, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,072 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985063) roll requested 2024-11-26T09:56:25,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985072 2024-11-26T09:56:25,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985063 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985072 2024-11-26T09:56:25,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,078 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=585, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,078 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985072) roll requested 2024-11-26T09:56:25,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985078 2024-11-26T09:56:25,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985072 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985078 2024-11-26T09:56:25,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,085 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=586, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,085 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985078) roll requested 2024-11-26T09:56:25,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985085 2024-11-26T09:56:25,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985078 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985085 2024-11-26T09:56:25,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,091 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,091 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=587, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,091 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985085) roll requested 2024-11-26T09:56:25,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985091 2024-11-26T09:56:25,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985091 2024-11-26T09:56:25,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,098 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=588, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985091) roll requested 2024-11-26T09:56:25,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985098 2024-11-26T09:56:25,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985091 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985098 2024-11-26T09:56:25,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=589, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985098) roll requested 2024-11-26T09:56:25,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985104 2024-11-26T09:56:25,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985098 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985104 2024-11-26T09:56:25,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,110 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,110 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=590, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,110 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985104) roll requested 2024-11-26T09:56:25,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985110 2024-11-26T09:56:25,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985110 2024-11-26T09:56:25,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,117 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,117 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=591, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,117 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985110) roll requested 2024-11-26T09:56:25,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985117 2024-11-26T09:56:25,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985110 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985117 2024-11-26T09:56:25,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,123 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,123 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=592, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,123 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985117) roll requested 2024-11-26T09:56:25,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985123 2024-11-26T09:56:25,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985123 2024-11-26T09:56:25,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,129 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=593, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,129 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985123) roll requested 2024-11-26T09:56:25,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985129 2024-11-26T09:56:25,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985123 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985129 2024-11-26T09:56:25,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,136 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=594, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,136 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985129) roll requested 2024-11-26T09:56:25,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985136 2024-11-26T09:56:25,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985129 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985136 2024-11-26T09:56:25,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,142 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,142 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=595, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,142 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985136) roll requested 2024-11-26T09:56:25,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985142 2024-11-26T09:56:25,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985136 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985142 2024-11-26T09:56:25,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=596, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985142) roll requested 2024-11-26T09:56:25,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985149 2024-11-26T09:56:25,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985142 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985149 2024-11-26T09:56:25,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,156 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=597, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985149) roll requested 2024-11-26T09:56:25,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985156 2024-11-26T09:56:25,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985156 2024-11-26T09:56:25,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=598, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,163 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985156) roll requested 2024-11-26T09:56:25,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985163 2024-11-26T09:56:25,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985163 2024-11-26T09:56:25,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,169 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,169 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=599, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,169 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985163) roll requested 2024-11-26T09:56:25,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985169 2024-11-26T09:56:25,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985163 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985169 2024-11-26T09:56:25,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,176 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,176 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=600, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,176 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985169) roll requested 2024-11-26T09:56:25,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985176 2024-11-26T09:56:25,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985169 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985176 2024-11-26T09:56:25,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,182 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,182 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=601, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,182 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985176) roll requested 2024-11-26T09:56:25,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985182 2024-11-26T09:56:25,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985176 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985182 2024-11-26T09:56:25,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,189 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=602, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985182) roll requested 2024-11-26T09:56:25,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985189 2024-11-26T09:56:25,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985182 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985189 2024-11-26T09:56:25,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=603, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985189) roll requested 2024-11-26T09:56:25,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985195 2024-11-26T09:56:25,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985195 2024-11-26T09:56:25,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,201 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,202 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=604, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,202 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985195) roll requested 2024-11-26T09:56:25,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985202 2024-11-26T09:56:25,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985195 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985202 2024-11-26T09:56:25,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=605, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,209 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985202) roll requested 2024-11-26T09:56:25,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985209 2024-11-26T09:56:25,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985202 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985209 2024-11-26T09:56:25,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,215 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,215 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=606, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,215 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985209) roll requested 2024-11-26T09:56:25,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985215 2024-11-26T09:56:25,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985215 2024-11-26T09:56:25,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,222 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,222 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=607, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,222 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985215) roll requested 2024-11-26T09:56:25,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985222 2024-11-26T09:56:25,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985215 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985222 2024-11-26T09:56:25,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,228 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,228 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=608, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,228 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985222) roll requested 2024-11-26T09:56:25,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985228 2024-11-26T09:56:25,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985222 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985228 2024-11-26T09:56:25,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,234 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,235 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=609, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,235 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985228) roll requested 2024-11-26T09:56:25,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985235 2024-11-26T09:56:25,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985228 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985235 2024-11-26T09:56:25,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,246 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=610, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985235) roll requested 2024-11-26T09:56:25,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985247 2024-11-26T09:56:25,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985235 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985247 2024-11-26T09:56:25,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,260 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,260 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=611, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,260 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985247) roll requested 2024-11-26T09:56:25,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985260 2024-11-26T09:56:25,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985247 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985260 2024-11-26T09:56:25,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=612, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,272 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985260) roll requested 2024-11-26T09:56:25,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985272 2024-11-26T09:56:25,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985260 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985272 2024-11-26T09:56:25,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,280 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,281 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=613, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,281 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985272) roll requested 2024-11-26T09:56:25,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985281 2024-11-26T09:56:25,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985281 2024-11-26T09:56:25,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=614, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985281) roll requested 2024-11-26T09:56:25,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985292 2024-11-26T09:56:25,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985281 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985292 2024-11-26T09:56:25,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,300 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,300 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=615, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,300 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985292) roll requested 2024-11-26T09:56:25,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985300 2024-11-26T09:56:25,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985300 2024-11-26T09:56:25,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,309 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=616, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,310 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985300) roll requested 2024-11-26T09:56:25,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985310 2024-11-26T09:56:25,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985300 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985310 2024-11-26T09:56:25,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,319 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,319 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=617, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985310) roll requested 2024-11-26T09:56:25,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985320 2024-11-26T09:56:25,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985310 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985320 2024-11-26T09:56:25,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,326 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,326 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=618, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,326 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985320) roll requested 2024-11-26T09:56:25,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985327 2024-11-26T09:56:25,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985320 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985327 2024-11-26T09:56:25,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,333 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,333 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=619, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,333 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985327) roll requested 2024-11-26T09:56:25,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985333 2024-11-26T09:56:25,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985327 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985333 2024-11-26T09:56:25,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,340 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=620, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,340 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985333) roll requested 2024-11-26T09:56:25,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985340 2024-11-26T09:56:25,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985333 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985340 2024-11-26T09:56:25,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,347 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,347 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=621, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,347 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985340) roll requested 2024-11-26T09:56:25,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985347 2024-11-26T09:56:25,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985340 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985347 2024-11-26T09:56:25,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,354 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,354 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=622, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985347) roll requested 2024-11-26T09:56:25,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985355 2024-11-26T09:56:25,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985347 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985355 2024-11-26T09:56:25,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=623, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985355) roll requested 2024-11-26T09:56:25,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985364 2024-11-26T09:56:25,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985364 2024-11-26T09:56:25,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,371 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,371 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=624, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,371 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985364) roll requested 2024-11-26T09:56:25,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985371 2024-11-26T09:56:25,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985371 2024-11-26T09:56:25,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,383 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,383 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=625, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,383 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985371) roll requested 2024-11-26T09:56:25,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985383 2024-11-26T09:56:25,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985383 2024-11-26T09:56:25,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,390 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,390 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=626, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,390 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985383) roll requested 2024-11-26T09:56:25,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985390 2024-11-26T09:56:25,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985383 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985390 2024-11-26T09:56:25,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,398 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,398 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=627, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,398 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985390) roll requested 2024-11-26T09:56:25,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985398 2024-11-26T09:56:25,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985390 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985398 2024-11-26T09:56:25,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,406 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,406 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=628, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,406 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985398) roll requested 2024-11-26T09:56:25,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985406 2024-11-26T09:56:25,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985398 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985406 2024-11-26T09:56:25,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,419 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=629, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985406) roll requested 2024-11-26T09:56:25,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985420 2024-11-26T09:56:25,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985406 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985420 2024-11-26T09:56:25,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,429 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,429 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=630, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,429 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985420) roll requested 2024-11-26T09:56:25,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985429 2024-11-26T09:56:25,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985420 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985429 2024-11-26T09:56:25,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,442 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,442 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=631, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,442 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985429) roll requested 2024-11-26T09:56:25,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985442 2024-11-26T09:56:25,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985429 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985442 2024-11-26T09:56:25,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,452 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,452 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=632, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,452 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985442) roll requested 2024-11-26T09:56:25,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985452 2024-11-26T09:56:25,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985452 2024-11-26T09:56:25,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,461 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,461 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=633, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,461 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985452) roll requested 2024-11-26T09:56:25,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985461 2024-11-26T09:56:25,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985452 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985461 2024-11-26T09:56:25,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,469 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,469 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=634, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,469 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985461) roll requested 2024-11-26T09:56:25,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985469 2024-11-26T09:56:25,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985469 2024-11-26T09:56:25,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,475 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,475 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=635, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,475 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985469) roll requested 2024-11-26T09:56:25,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985475 2024-11-26T09:56:25,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985469 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985475 2024-11-26T09:56:25,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=636, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985475) roll requested 2024-11-26T09:56:25,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985485 2024-11-26T09:56:25,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985475 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985485 2024-11-26T09:56:25,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,497 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,497 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=637, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985485) roll requested 2024-11-26T09:56:25,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985497 2024-11-26T09:56:25,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985497 2024-11-26T09:56:25,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,508 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=638, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,508 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985497) roll requested 2024-11-26T09:56:25,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985508 2024-11-26T09:56:25,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985508 2024-11-26T09:56:25,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,516 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,516 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=639, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,516 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985508) roll requested 2024-11-26T09:56:25,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985516 2024-11-26T09:56:25,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985508 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985516 2024-11-26T09:56:25,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,528 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=640, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,528 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985516) roll requested 2024-11-26T09:56:25,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985528 2024-11-26T09:56:25,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985516 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985528 2024-11-26T09:56:25,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,539 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,539 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=641, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,539 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985528) roll requested 2024-11-26T09:56:25,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985539 2024-11-26T09:56:25,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985528 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985539 2024-11-26T09:56:25,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,547 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,547 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=642, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,548 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985539) roll requested 2024-11-26T09:56:25,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985548 2024-11-26T09:56:25,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985539 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985548 2024-11-26T09:56:25,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,555 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,556 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=643, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,556 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985548) roll requested 2024-11-26T09:56:25,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985556 2024-11-26T09:56:25,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985548 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985556 2024-11-26T09:56:25,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,563 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,563 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=644, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,563 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985556) roll requested 2024-11-26T09:56:25,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985563 2024-11-26T09:56:25,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985556 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985563 2024-11-26T09:56:25,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,569 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,570 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=645, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,570 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985563) roll requested 2024-11-26T09:56:25,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985570 2024-11-26T09:56:25,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985563 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985570 2024-11-26T09:56:25,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,576 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,576 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=646, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,576 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985570) roll requested 2024-11-26T09:56:25,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985577 2024-11-26T09:56:25,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985570 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985577 2024-11-26T09:56:25,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,583 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,583 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=647, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,583 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985577) roll requested 2024-11-26T09:56:25,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985584 2024-11-26T09:56:25,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985577 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985584 2024-11-26T09:56:25,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,591 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,591 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=648, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,591 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985584) roll requested 2024-11-26T09:56:25,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985591 2024-11-26T09:56:25,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985584 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985591 2024-11-26T09:56:25,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,599 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,599 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=649, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,599 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985591) roll requested 2024-11-26T09:56:25,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985599 2024-11-26T09:56:25,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985591 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985599 2024-11-26T09:56:25,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=650, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,607 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985599) roll requested 2024-11-26T09:56:25,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985608 2024-11-26T09:56:25,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985599 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985608 2024-11-26T09:56:25,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,615 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,615 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=651, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,616 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985608) roll requested 2024-11-26T09:56:25,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985616 2024-11-26T09:56:25,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985616 2024-11-26T09:56:25,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,627 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,628 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=652, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985616) roll requested 2024-11-26T09:56:25,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985628 2024-11-26T09:56:25,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985616 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985628 2024-11-26T09:56:25,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,635 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,635 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=653, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,635 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985628) roll requested 2024-11-26T09:56:25,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985635 2024-11-26T09:56:25,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985635 2024-11-26T09:56:25,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,644 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=654, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,644 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985635) roll requested 2024-11-26T09:56:25,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985644 2024-11-26T09:56:25,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985635 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985644 2024-11-26T09:56:25,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=655, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,652 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985644) roll requested 2024-11-26T09:56:25,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985652 2024-11-26T09:56:25,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985644 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985652 2024-11-26T09:56:25,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,660 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,660 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=656, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,660 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985652) roll requested 2024-11-26T09:56:25,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985660 2024-11-26T09:56:25,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985652 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985660 2024-11-26T09:56:25,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,668 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,668 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=657, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,668 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985660) roll requested 2024-11-26T09:56:25,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985668 2024-11-26T09:56:25,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985660 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985668 2024-11-26T09:56:25,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,676 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,676 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=658, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,677 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985668) roll requested 2024-11-26T09:56:25,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985677 2024-11-26T09:56:25,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985668 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985677 2024-11-26T09:56:25,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,684 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,684 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=659, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,684 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985677) roll requested 2024-11-26T09:56:25,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985684 2024-11-26T09:56:25,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985677 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985684 2024-11-26T09:56:25,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,691 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,691 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=660, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,691 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985684) roll requested 2024-11-26T09:56:25,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985691 2024-11-26T09:56:25,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985684 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985691 2024-11-26T09:56:25,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,698 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,698 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=661, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,698 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985691) roll requested 2024-11-26T09:56:25,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985698 2024-11-26T09:56:25,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985698 2024-11-26T09:56:25,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=662, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985698) roll requested 2024-11-26T09:56:25,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985708 2024-11-26T09:56:25,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985698 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985708 2024-11-26T09:56:25,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,715 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=663, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985708) roll requested 2024-11-26T09:56:25,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985716 2024-11-26T09:56:25,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985716 2024-11-26T09:56:25,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,722 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,723 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=664, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,723 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985716) roll requested 2024-11-26T09:56:25,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985723 2024-11-26T09:56:25,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985723 2024-11-26T09:56:25,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,729 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=665, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985723) roll requested 2024-11-26T09:56:25,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985730 2024-11-26T09:56:25,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985723 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985730 2024-11-26T09:56:25,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,736 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,737 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=666, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,737 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985730) roll requested 2024-11-26T09:56:25,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985737 2024-11-26T09:56:25,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985737 2024-11-26T09:56:25,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,746 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=667, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,746 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985737) roll requested 2024-11-26T09:56:25,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985746 2024-11-26T09:56:25,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985737 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985746 2024-11-26T09:56:25,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,755 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,755 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=668, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,755 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985746) roll requested 2024-11-26T09:56:25,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985755 2024-11-26T09:56:25,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985746 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985755 2024-11-26T09:56:25,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,771 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,771 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=669, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,771 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985755) roll requested 2024-11-26T09:56:25,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985771 2024-11-26T09:56:25,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985755 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985771 2024-11-26T09:56:25,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,778 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,778 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=670, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,778 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985771) roll requested 2024-11-26T09:56:25,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985778 2024-11-26T09:56:25,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985771 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985778 2024-11-26T09:56:25,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,786 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=671, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,786 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985778) roll requested 2024-11-26T09:56:25,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985786 2024-11-26T09:56:25,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985778 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985786 2024-11-26T09:56:25,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,792 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=672, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985786) roll requested 2024-11-26T09:56:25,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985792 2024-11-26T09:56:25,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985786 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985792 2024-11-26T09:56:25,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,800 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=673, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,800 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985792) roll requested 2024-11-26T09:56:25,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985800 2024-11-26T09:56:25,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985792 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985800 2024-11-26T09:56:25,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=674, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,806 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985800) roll requested 2024-11-26T09:56:25,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985806 2024-11-26T09:56:25,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985800 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985806 2024-11-26T09:56:25,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,813 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=675, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,813 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985806) roll requested 2024-11-26T09:56:25,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985813 2024-11-26T09:56:25,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985806 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985813 2024-11-26T09:56:25,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=676, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985813) roll requested 2024-11-26T09:56:25,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985821 2024-11-26T09:56:25,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985821 2024-11-26T09:56:25,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=677, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985821) roll requested 2024-11-26T09:56:25,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985830 2024-11-26T09:56:25,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985830 2024-11-26T09:56:25,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,838 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,839 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=678, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985830) roll requested 2024-11-26T09:56:25,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985839 2024-11-26T09:56:25,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985839 2024-11-26T09:56:25,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,846 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=679, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,846 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985839) roll requested 2024-11-26T09:56:25,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985846 2024-11-26T09:56:25,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985846 2024-11-26T09:56:25,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,854 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,854 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=680, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985846) roll requested 2024-11-26T09:56:25,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985854 2024-11-26T09:56:25,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985846 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985854 2024-11-26T09:56:25,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,861 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,862 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=681, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,862 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985854) roll requested 2024-11-26T09:56:25,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985862 2024-11-26T09:56:25,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985862 2024-11-26T09:56:25,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,868 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,868 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=682, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,869 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985862) roll requested 2024-11-26T09:56:25,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985869 2024-11-26T09:56:25,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985862 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985869 2024-11-26T09:56:25,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,876 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,877 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=683, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,877 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985869) roll requested 2024-11-26T09:56:25,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985877 2024-11-26T09:56:25,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985869 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985877 2024-11-26T09:56:25,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,883 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,884 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=684, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,884 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985877) roll requested 2024-11-26T09:56:25,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985884 2024-11-26T09:56:25,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985877 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985884 2024-11-26T09:56:25,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,894 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=685, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,894 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985884) roll requested 2024-11-26T09:56:25,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985894 2024-11-26T09:56:25,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985884 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985894 2024-11-26T09:56:25,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,904 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,904 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=686, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,904 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985894) roll requested 2024-11-26T09:56:25,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985904 2024-11-26T09:56:25,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985904 2024-11-26T09:56:25,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,913 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,913 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=687, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,914 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985904) roll requested 2024-11-26T09:56:25,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985914 2024-11-26T09:56:25,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985904 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985914 2024-11-26T09:56:25,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,925 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,925 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=688, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,925 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985914) roll requested 2024-11-26T09:56:25,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985925 2024-11-26T09:56:25,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985914 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985925 2024-11-26T09:56:25,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,936 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=689, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,936 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985925) roll requested 2024-11-26T09:56:25,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985936 2024-11-26T09:56:25,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985925 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985936 2024-11-26T09:56:25,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,949 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=690, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985936) roll requested 2024-11-26T09:56:25,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985949 2024-11-26T09:56:25,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985936 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985949 2024-11-26T09:56:25,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,957 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,957 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=691, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,958 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985949) roll requested 2024-11-26T09:56:25,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985958 2024-11-26T09:56:25,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985958 2024-11-26T09:56:25,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,968 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,968 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=692, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,968 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985958) roll requested 2024-11-26T09:56:25,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985968 2024-11-26T09:56:25,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985958 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985968 2024-11-26T09:56:25,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,984 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:25,985 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=693, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:25,985 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:25,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985968) roll requested 2024-11-26T09:56:25,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614985985 2024-11-26T09:56:25,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:25,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985968 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985985 2024-11-26T09:56:25,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:25,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,000 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=694, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,000 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614985985) roll requested 2024-11-26T09:56:26,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986000 2024-11-26T09:56:26,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614985985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986000 2024-11-26T09:56:26,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,032 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,032 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=695, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,032 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986000) roll requested 2024-11-26T09:56:26,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986032 2024-11-26T09:56:26,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986000 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986032 2024-11-26T09:56:26,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=696, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986032) roll requested 2024-11-26T09:56:26,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986048 2024-11-26T09:56:26,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986032 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986048 2024-11-26T09:56:26,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,057 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,058 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=697, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,058 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986048) roll requested 2024-11-26T09:56:26,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986058 2024-11-26T09:56:26,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986058 2024-11-26T09:56:26,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,068 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,068 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=698, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,068 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986058) roll requested 2024-11-26T09:56:26,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986068 2024-11-26T09:56:26,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986058 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986068 2024-11-26T09:56:26,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,077 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=699, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,077 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986068) roll requested 2024-11-26T09:56:26,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986077 2024-11-26T09:56:26,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986068 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986077 2024-11-26T09:56:26,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,085 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,085 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=700, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,085 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986077) roll requested 2024-11-26T09:56:26,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986085 2024-11-26T09:56:26,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986077 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986085 2024-11-26T09:56:26,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,092 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=701, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,092 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986085) roll requested 2024-11-26T09:56:26,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986092 2024-11-26T09:56:26,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986092 2024-11-26T09:56:26,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,098 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,099 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=702, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,099 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986092) roll requested 2024-11-26T09:56:26,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986099 2024-11-26T09:56:26,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986092 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986099 2024-11-26T09:56:26,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,105 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,105 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=703, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986099) roll requested 2024-11-26T09:56:26,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986105 2024-11-26T09:56:26,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986099 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986105 2024-11-26T09:56:26,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,111 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,112 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=704, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,112 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986105) roll requested 2024-11-26T09:56:26,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986112 2024-11-26T09:56:26,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986112 2024-11-26T09:56:26,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,118 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,118 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=705, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,118 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986112) roll requested 2024-11-26T09:56:26,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986118 2024-11-26T09:56:26,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986112 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986118 2024-11-26T09:56:26,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=706, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986118) roll requested 2024-11-26T09:56:26,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986126 2024-11-26T09:56:26,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986118 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986126 2024-11-26T09:56:26,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,132 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,132 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=707, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,132 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986126) roll requested 2024-11-26T09:56:26,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986132 2024-11-26T09:56:26,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986126 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986132 2024-11-26T09:56:26,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,138 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,138 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=708, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,139 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986132) roll requested 2024-11-26T09:56:26,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986139 2024-11-26T09:56:26,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986132 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986139 2024-11-26T09:56:26,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,145 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,145 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=709, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,145 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986139) roll requested 2024-11-26T09:56:26,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986145 2024-11-26T09:56:26,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986139 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986145 2024-11-26T09:56:26,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,151 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,152 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=710, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,152 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986145) roll requested 2024-11-26T09:56:26,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986152 2024-11-26T09:56:26,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986145 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986152 2024-11-26T09:56:26,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,158 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,158 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=711, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,158 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986152) roll requested 2024-11-26T09:56:26,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986158 2024-11-26T09:56:26,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986152 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986158 2024-11-26T09:56:26,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,164 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,164 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=712, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,164 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986158) roll requested 2024-11-26T09:56:26,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986164 2024-11-26T09:56:26,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986158 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986164 2024-11-26T09:56:26,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,170 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,170 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=713, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,171 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986164) roll requested 2024-11-26T09:56:26,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986171 2024-11-26T09:56:26,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986164 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986171 2024-11-26T09:56:26,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,177 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,177 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=714, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,177 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986171) roll requested 2024-11-26T09:56:26,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986177 2024-11-26T09:56:26,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986171 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986177 2024-11-26T09:56:26,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,183 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,183 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=715, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,183 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986177) roll requested 2024-11-26T09:56:26,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986183 2024-11-26T09:56:26,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986183 2024-11-26T09:56:26,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,189 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,190 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=716, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,190 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986183) roll requested 2024-11-26T09:56:26,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986190 2024-11-26T09:56:26,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986183 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986190 2024-11-26T09:56:26,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=717, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986190) roll requested 2024-11-26T09:56:26,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986196 2024-11-26T09:56:26,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986190 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986196 2024-11-26T09:56:26,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,202 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,202 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=718, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,202 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986196) roll requested 2024-11-26T09:56:26,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986202 2024-11-26T09:56:26,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986202 2024-11-26T09:56:26,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=719, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986202) roll requested 2024-11-26T09:56:26,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986210 2024-11-26T09:56:26,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986202 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986210 2024-11-26T09:56:26,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,219 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,219 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=720, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986210) roll requested 2024-11-26T09:56:26,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986219 2024-11-26T09:56:26,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986219 2024-11-26T09:56:26,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,227 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,227 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=721, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,227 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986219) roll requested 2024-11-26T09:56:26,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986227 2024-11-26T09:56:26,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986227 2024-11-26T09:56:26,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,234 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,234 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=722, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,234 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986227) roll requested 2024-11-26T09:56:26,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986234 2024-11-26T09:56:26,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986227 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986234 2024-11-26T09:56:26,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=723, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,244 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986234) roll requested 2024-11-26T09:56:26,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986244 2024-11-26T09:56:26,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986234 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986244 2024-11-26T09:56:26,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=724, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986244) roll requested 2024-11-26T09:56:26,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986253 2024-11-26T09:56:26,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986244 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986253 2024-11-26T09:56:26,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,260 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,260 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=725, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986253) roll requested 2024-11-26T09:56:26,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986261 2024-11-26T09:56:26,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986261 2024-11-26T09:56:26,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=726, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986261) roll requested 2024-11-26T09:56:26,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986268 2024-11-26T09:56:26,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986268 2024-11-26T09:56:26,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,275 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,275 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=727, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,275 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986268) roll requested 2024-11-26T09:56:26,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986275 2024-11-26T09:56:26,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986268 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986275 2024-11-26T09:56:26,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,282 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,282 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=728, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,282 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986275) roll requested 2024-11-26T09:56:26,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986283 2024-11-26T09:56:26,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986275 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986283 2024-11-26T09:56:26,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,289 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,289 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=729, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,289 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986283) roll requested 2024-11-26T09:56:26,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986289 2024-11-26T09:56:26,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986283 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986289 2024-11-26T09:56:26,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=730, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,296 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986289) roll requested 2024-11-26T09:56:26,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986296 2024-11-26T09:56:26,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986289 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986296 2024-11-26T09:56:26,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,303 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=731, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,303 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986296) roll requested 2024-11-26T09:56:26,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986303 2024-11-26T09:56:26,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986296 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986303 2024-11-26T09:56:26,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,311 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=732, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986303) roll requested 2024-11-26T09:56:26,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986312 2024-11-26T09:56:26,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986303 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986312 2024-11-26T09:56:26,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,318 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,318 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=733, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,318 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986312) roll requested 2024-11-26T09:56:26,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986318 2024-11-26T09:56:26,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986318 2024-11-26T09:56:26,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,325 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,325 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=734, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,325 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986318) roll requested 2024-11-26T09:56:26,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986325 2024-11-26T09:56:26,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986318 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986325 2024-11-26T09:56:26,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,331 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,331 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=735, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,331 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986325) roll requested 2024-11-26T09:56:26,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986331 2024-11-26T09:56:26,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986331 2024-11-26T09:56:26,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,337 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,338 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=736, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,338 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986331) roll requested 2024-11-26T09:56:26,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986338 2024-11-26T09:56:26,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986331 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986338 2024-11-26T09:56:26,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,344 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=737, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,344 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986338) roll requested 2024-11-26T09:56:26,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986344 2024-11-26T09:56:26,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986338 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986344 2024-11-26T09:56:26,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,350 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,351 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=738, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,351 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986344) roll requested 2024-11-26T09:56:26,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986351 2024-11-26T09:56:26,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986344 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986351 2024-11-26T09:56:26,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,357 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,357 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=739, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,357 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986351) roll requested 2024-11-26T09:56:26,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986357 2024-11-26T09:56:26,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986351 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986357 2024-11-26T09:56:26,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,363 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,363 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=740, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,363 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986357) roll requested 2024-11-26T09:56:26,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986363 2024-11-26T09:56:26,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986357 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986363 2024-11-26T09:56:26,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,370 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,370 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=741, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,370 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986363) roll requested 2024-11-26T09:56:26,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986370 2024-11-26T09:56:26,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986363 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986370 2024-11-26T09:56:26,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,376 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,377 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=742, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,377 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986370) roll requested 2024-11-26T09:56:26,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986377 2024-11-26T09:56:26,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986377 2024-11-26T09:56:26,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,383 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,383 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=743, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,383 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986377) roll requested 2024-11-26T09:56:26,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986383 2024-11-26T09:56:26,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986377 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986383 2024-11-26T09:56:26,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,389 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,389 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=744, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,390 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986383) roll requested 2024-11-26T09:56:26,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986390 2024-11-26T09:56:26,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986383 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986390 2024-11-26T09:56:26,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,396 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,396 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=745, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,396 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986390) roll requested 2024-11-26T09:56:26,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986396 2024-11-26T09:56:26,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986390 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986396 2024-11-26T09:56:26,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,402 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,402 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=746, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,402 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986396) roll requested 2024-11-26T09:56:26,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986402 2024-11-26T09:56:26,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986396 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986402 2024-11-26T09:56:26,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,409 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,409 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=747, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,409 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986402) roll requested 2024-11-26T09:56:26,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986409 2024-11-26T09:56:26,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986402 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986409 2024-11-26T09:56:26,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,415 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=748, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986409) roll requested 2024-11-26T09:56:26,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986416 2024-11-26T09:56:26,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986409 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986416 2024-11-26T09:56:26,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,422 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,422 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=749, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,422 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986416) roll requested 2024-11-26T09:56:26,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986422 2024-11-26T09:56:26,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986416 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986422 2024-11-26T09:56:26,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,428 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=750, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986422) roll requested 2024-11-26T09:56:26,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986428 2024-11-26T09:56:26,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986422 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986428 2024-11-26T09:56:26,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,435 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,435 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=751, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,435 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986428) roll requested 2024-11-26T09:56:26,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986435 2024-11-26T09:56:26,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986435 2024-11-26T09:56:26,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,442 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,442 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=752, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,442 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986435) roll requested 2024-11-26T09:56:26,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986442 2024-11-26T09:56:26,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986435 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986442 2024-11-26T09:56:26,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,453 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,454 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=753, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,454 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986442) roll requested 2024-11-26T09:56:26,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986454 2024-11-26T09:56:26,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986454 2024-11-26T09:56:26,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,461 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,462 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=754, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,462 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986454) roll requested 2024-11-26T09:56:26,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986462 2024-11-26T09:56:26,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986454 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986462 2024-11-26T09:56:26,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,469 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,469 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=755, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,469 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986462) roll requested 2024-11-26T09:56:26,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986470 2024-11-26T09:56:26,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986470 2024-11-26T09:56:26,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,477 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=756, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986470) roll requested 2024-11-26T09:56:26,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986477 2024-11-26T09:56:26,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986470 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986477 2024-11-26T09:56:26,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,492 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=757, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986477) roll requested 2024-11-26T09:56:26,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986492 2024-11-26T09:56:26,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986492 2024-11-26T09:56:26,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,499 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,500 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=758, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,500 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986492) roll requested 2024-11-26T09:56:26,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986500 2024-11-26T09:56:26,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986500 2024-11-26T09:56:26,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,508 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=759, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,508 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986500) roll requested 2024-11-26T09:56:26,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986509 2024-11-26T09:56:26,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986500 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986509 2024-11-26T09:56:26,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,520 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=760, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,520 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986509) roll requested 2024-11-26T09:56:26,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986520 2024-11-26T09:56:26,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986520 2024-11-26T09:56:26,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,527 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,527 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=761, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,527 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986520) roll requested 2024-11-26T09:56:26,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986527 2024-11-26T09:56:26,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986520 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986527 2024-11-26T09:56:26,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,533 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,533 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=762, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,533 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986527) roll requested 2024-11-26T09:56:26,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986533 2024-11-26T09:56:26,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986527 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986533 2024-11-26T09:56:26,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,540 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,540 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=763, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,540 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986533) roll requested 2024-11-26T09:56:26,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986540 2024-11-26T09:56:26,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986533 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986540 2024-11-26T09:56:26,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,546 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,547 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=764, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,547 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986540) roll requested 2024-11-26T09:56:26,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986547 2024-11-26T09:56:26,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986540 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986547 2024-11-26T09:56:26,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,553 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,553 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=765, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,553 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986547) roll requested 2024-11-26T09:56:26,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986553 2024-11-26T09:56:26,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986547 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986553 2024-11-26T09:56:26,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,559 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,559 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=766, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,559 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986553) roll requested 2024-11-26T09:56:26,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986560 2024-11-26T09:56:26,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986553 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986560 2024-11-26T09:56:26,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,566 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,566 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=767, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,566 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986560) roll requested 2024-11-26T09:56:26,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986566 2024-11-26T09:56:26,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986560 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986566 2024-11-26T09:56:26,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,572 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=768, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,572 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986566) roll requested 2024-11-26T09:56:26,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986572 2024-11-26T09:56:26,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986566 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986572 2024-11-26T09:56:26,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,582 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=769, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,582 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986572) roll requested 2024-11-26T09:56:26,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986582 2024-11-26T09:56:26,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986572 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986582 2024-11-26T09:56:26,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,592 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,592 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=770, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,592 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986582) roll requested 2024-11-26T09:56:26,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986592 2024-11-26T09:56:26,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986582 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986592 2024-11-26T09:56:26,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,599 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,599 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=771, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,599 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986592) roll requested 2024-11-26T09:56:26,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986599 2024-11-26T09:56:26,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986592 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986599 2024-11-26T09:56:26,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=772, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986599) roll requested 2024-11-26T09:56:26,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986608 2024-11-26T09:56:26,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986599 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986608 2024-11-26T09:56:26,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,616 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,616 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=773, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,616 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986608) roll requested 2024-11-26T09:56:26,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986616 2024-11-26T09:56:26,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986616 2024-11-26T09:56:26,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,625 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,625 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=774, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,625 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986616) roll requested 2024-11-26T09:56:26,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986625 2024-11-26T09:56:26,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986616 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986625 2024-11-26T09:56:26,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,632 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=775, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986625) roll requested 2024-11-26T09:56:26,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986633 2024-11-26T09:56:26,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986633 2024-11-26T09:56:26,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,641 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,641 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=776, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,641 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986633) roll requested 2024-11-26T09:56:26,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986641 2024-11-26T09:56:26,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986641 2024-11-26T09:56:26,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,647 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,648 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=777, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,648 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986641) roll requested 2024-11-26T09:56:26,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986648 2024-11-26T09:56:26,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986641 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986648 2024-11-26T09:56:26,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,654 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,654 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=778, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,654 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986648) roll requested 2024-11-26T09:56:26,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986655 2024-11-26T09:56:26,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986648 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986655 2024-11-26T09:56:26,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,661 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,661 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=779, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,661 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986655) roll requested 2024-11-26T09:56:26,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986661 2024-11-26T09:56:26,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986655 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986661 2024-11-26T09:56:26,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,667 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,668 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=780, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,668 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986661) roll requested 2024-11-26T09:56:26,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986668 2024-11-26T09:56:26,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986661 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986668 2024-11-26T09:56:26,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=781, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986668) roll requested 2024-11-26T09:56:26,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986674 2024-11-26T09:56:26,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986668 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986674 2024-11-26T09:56:26,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,681 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=782, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986674) roll requested 2024-11-26T09:56:26,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986681 2024-11-26T09:56:26,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986681 2024-11-26T09:56:26,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,687 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=783, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,688 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986681) roll requested 2024-11-26T09:56:26,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986688 2024-11-26T09:56:26,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986688 2024-11-26T09:56:26,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,694 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,694 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=784, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,694 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986688) roll requested 2024-11-26T09:56:26,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986694 2024-11-26T09:56:26,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986688 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986694 2024-11-26T09:56:26,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,702 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=785, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,702 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986694) roll requested 2024-11-26T09:56:26,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986702 2024-11-26T09:56:26,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986702 2024-11-26T09:56:26,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=786, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986702) roll requested 2024-11-26T09:56:26,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986709 2024-11-26T09:56:26,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986702 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986709 2024-11-26T09:56:26,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,715 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=787, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,715 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986709) roll requested 2024-11-26T09:56:26,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986715 2024-11-26T09:56:26,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986715 2024-11-26T09:56:26,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,722 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,722 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=788, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,722 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986715) roll requested 2024-11-26T09:56:26,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986722 2024-11-26T09:56:26,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986715 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986722 2024-11-26T09:56:26,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,728 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,728 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=789, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,728 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986722) roll requested 2024-11-26T09:56:26,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986728 2024-11-26T09:56:26,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986722 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986728 2024-11-26T09:56:26,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,735 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,735 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=790, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,735 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986728) roll requested 2024-11-26T09:56:26,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986735 2024-11-26T09:56:26,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986728 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986735 2024-11-26T09:56:26,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,741 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,742 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=791, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,742 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986735) roll requested 2024-11-26T09:56:26,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986742 2024-11-26T09:56:26,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986735 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986742 2024-11-26T09:56:26,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=792, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,748 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986742) roll requested 2024-11-26T09:56:26,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986748 2024-11-26T09:56:26,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986742 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986748 2024-11-26T09:56:26,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,754 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,755 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=793, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,755 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986748) roll requested 2024-11-26T09:56:26,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986755 2024-11-26T09:56:26,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986748 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986755 2024-11-26T09:56:26,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,761 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=794, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,761 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986755) roll requested 2024-11-26T09:56:26,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986761 2024-11-26T09:56:26,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986755 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986761 2024-11-26T09:56:26,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,767 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=795, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986761) roll requested 2024-11-26T09:56:26,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986768 2024-11-26T09:56:26,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986768 2024-11-26T09:56:26,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,774 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,774 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=796, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,774 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986768) roll requested 2024-11-26T09:56:26,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986774 2024-11-26T09:56:26,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986768 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986774 2024-11-26T09:56:26,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,781 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,781 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=797, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,781 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986774) roll requested 2024-11-26T09:56:26,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986781 2024-11-26T09:56:26,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986774 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986781 2024-11-26T09:56:26,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,787 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=798, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986781) roll requested 2024-11-26T09:56:26,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986788 2024-11-26T09:56:26,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986781 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986788 2024-11-26T09:56:26,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,794 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,794 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=799, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,794 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986788) roll requested 2024-11-26T09:56:26,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986794 2024-11-26T09:56:26,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986788 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986794 2024-11-26T09:56:26,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,802 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=800, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986794) roll requested 2024-11-26T09:56:26,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986802 2024-11-26T09:56:26,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986794 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986802 2024-11-26T09:56:26,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,809 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=801, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,809 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986802) roll requested 2024-11-26T09:56:26,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986809 2024-11-26T09:56:26,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986809 2024-11-26T09:56:26,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,815 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,815 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=802, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986809) roll requested 2024-11-26T09:56:26,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986815 2024-11-26T09:56:26,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986809 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986815 2024-11-26T09:56:26,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,822 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=803, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,822 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986815) roll requested 2024-11-26T09:56:26,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986822 2024-11-26T09:56:26,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986822 2024-11-26T09:56:26,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=804, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986822) roll requested 2024-11-26T09:56:26,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986828 2024-11-26T09:56:26,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986822 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986828 2024-11-26T09:56:26,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,835 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,835 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=805, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,835 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986828) roll requested 2024-11-26T09:56:26,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986835 2024-11-26T09:56:26,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986835 2024-11-26T09:56:26,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,841 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,841 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=806, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,841 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986835) roll requested 2024-11-26T09:56:26,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986841 2024-11-26T09:56:26,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986835 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986841 2024-11-26T09:56:26,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,848 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=807, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,848 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986841) roll requested 2024-11-26T09:56:26,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986848 2024-11-26T09:56:26,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986841 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986848 2024-11-26T09:56:26,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,854 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,854 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=808, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986848) roll requested 2024-11-26T09:56:26,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986854 2024-11-26T09:56:26,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986848 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986854 2024-11-26T09:56:26,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,860 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,861 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=809, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,861 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986854) roll requested 2024-11-26T09:56:26,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986861 2024-11-26T09:56:26,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986861 2024-11-26T09:56:26,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,867 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,867 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=810, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,867 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986861) roll requested 2024-11-26T09:56:26,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986867 2024-11-26T09:56:26,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986861 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986867 2024-11-26T09:56:26,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=811, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986867) roll requested 2024-11-26T09:56:26,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986873 2024-11-26T09:56:26,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986867 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986873 2024-11-26T09:56:26,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=812, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986873) roll requested 2024-11-26T09:56:26,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986880 2024-11-26T09:56:26,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986880 2024-11-26T09:56:26,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,886 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,886 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=813, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,886 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986880) roll requested 2024-11-26T09:56:26,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986887 2024-11-26T09:56:26,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986887 2024-11-26T09:56:26,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=814, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986887) roll requested 2024-11-26T09:56:26,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986893 2024-11-26T09:56:26,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986887 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986893 2024-11-26T09:56:26,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,901 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=815, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,901 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986893) roll requested 2024-11-26T09:56:26,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986901 2024-11-26T09:56:26,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986901 2024-11-26T09:56:26,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,907 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,907 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=816, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,907 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986901) roll requested 2024-11-26T09:56:26,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986907 2024-11-26T09:56:26,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986907 2024-11-26T09:56:26,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=817, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,920 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986907) roll requested 2024-11-26T09:56:26,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986920 2024-11-26T09:56:26,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986907 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986920 2024-11-26T09:56:26,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,927 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,927 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=818, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,927 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986920) roll requested 2024-11-26T09:56:26,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986928 2024-11-26T09:56:26,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986928 2024-11-26T09:56:26,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,935 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,935 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=819, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,935 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986928) roll requested 2024-11-26T09:56:26,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986935 2024-11-26T09:56:26,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986935 2024-11-26T09:56:26,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,941 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,942 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=820, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,942 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986935) roll requested 2024-11-26T09:56:26,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986942 2024-11-26T09:56:26,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986935 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986942 2024-11-26T09:56:26,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,948 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=821, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986942) roll requested 2024-11-26T09:56:26,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986949 2024-11-26T09:56:26,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986942 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986949 2024-11-26T09:56:26,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,956 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,956 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=822, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,956 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986949) roll requested 2024-11-26T09:56:26,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986956 2024-11-26T09:56:26,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986956 2024-11-26T09:56:26,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=823, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,964 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986956) roll requested 2024-11-26T09:56:26,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986964 2024-11-26T09:56:26,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986956 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986964 2024-11-26T09:56:26,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,970 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=824, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,971 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986964) roll requested 2024-11-26T09:56:26,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986971 2024-11-26T09:56:26,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986971 2024-11-26T09:56:26,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,980 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,980 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=825, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,980 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986971) roll requested 2024-11-26T09:56:26,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986980 2024-11-26T09:56:26,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986971 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986980 2024-11-26T09:56:26,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,986 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,987 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=826, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,987 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986980) roll requested 2024-11-26T09:56:26,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986987 2024-11-26T09:56:26,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986980 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986987 2024-11-26T09:56:26,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:26,993 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:26,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=827, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:26,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:26,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986987) roll requested 2024-11-26T09:56:26,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614986994 2024-11-26T09:56:26,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:26,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986994 2024-11-26T09:56:27,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,000 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,000 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=828, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,000 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614986994) roll requested 2024-11-26T09:56:27,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987000 2024-11-26T09:56:27,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614986994 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987000 2024-11-26T09:56:27,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,006 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=829, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987000) roll requested 2024-11-26T09:56:27,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987007 2024-11-26T09:56:27,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987000 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987007 2024-11-26T09:56:27,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,013 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=830, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,013 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987007) roll requested 2024-11-26T09:56:27,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987013 2024-11-26T09:56:27,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987013 2024-11-26T09:56:27,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,020 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,020 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=831, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,020 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987013) roll requested 2024-11-26T09:56:27,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987020 2024-11-26T09:56:27,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987013 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987020 2024-11-26T09:56:27,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,026 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,026 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=832, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,027 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987020) roll requested 2024-11-26T09:56:27,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987027 2024-11-26T09:56:27,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987020 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987027 2024-11-26T09:56:27,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,033 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,033 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=833, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,033 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987027) roll requested 2024-11-26T09:56:27,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987033 2024-11-26T09:56:27,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987027 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987033 2024-11-26T09:56:27,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,040 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,040 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=834, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,040 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987033) roll requested 2024-11-26T09:56:27,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987040 2024-11-26T09:56:27,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987033 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987040 2024-11-26T09:56:27,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,047 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,047 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=835, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,047 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987040) roll requested 2024-11-26T09:56:27,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987047 2024-11-26T09:56:27,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987040 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987047 2024-11-26T09:56:27,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,054 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=836, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987047) roll requested 2024-11-26T09:56:27,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987054 2024-11-26T09:56:27,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987047 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987054 2024-11-26T09:56:27,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,061 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=837, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987054) roll requested 2024-11-26T09:56:27,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987061 2024-11-26T09:56:27,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987054 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987061 2024-11-26T09:56:27,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,068 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,068 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=838, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,068 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987061) roll requested 2024-11-26T09:56:27,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987068 2024-11-26T09:56:27,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987068 2024-11-26T09:56:27,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,075 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,075 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=839, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,075 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987068) roll requested 2024-11-26T09:56:27,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987075 2024-11-26T09:56:27,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987068 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987075 2024-11-26T09:56:27,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,082 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,083 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=840, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,083 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987075) roll requested 2024-11-26T09:56:27,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987083 2024-11-26T09:56:27,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987075 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987083 2024-11-26T09:56:27,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,090 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,090 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=841, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,090 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987083) roll requested 2024-11-26T09:56:27,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987090 2024-11-26T09:56:27,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987083 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987090 2024-11-26T09:56:27,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=842, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987090) roll requested 2024-11-26T09:56:27,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987097 2024-11-26T09:56:27,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987090 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987097 2024-11-26T09:56:27,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=843, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987097) roll requested 2024-11-26T09:56:27,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987104 2024-11-26T09:56:27,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987104 2024-11-26T09:56:27,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,110 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,111 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=844, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,111 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987104) roll requested 2024-11-26T09:56:27,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987111 2024-11-26T09:56:27,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987111 2024-11-26T09:56:27,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,117 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,118 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=845, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,118 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987111) roll requested 2024-11-26T09:56:27,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987118 2024-11-26T09:56:27,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987111 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987118 2024-11-26T09:56:27,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,124 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,124 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=846, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,124 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987118) roll requested 2024-11-26T09:56:27,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987124 2024-11-26T09:56:27,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987118 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987124 2024-11-26T09:56:27,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,130 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,131 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=847, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,131 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987124) roll requested 2024-11-26T09:56:27,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987131 2024-11-26T09:56:27,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987124 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987131 2024-11-26T09:56:27,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,137 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,137 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=848, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,137 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987131) roll requested 2024-11-26T09:56:27,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987138 2024-11-26T09:56:27,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987131 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987138 2024-11-26T09:56:27,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,144 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,144 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=849, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,144 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987138) roll requested 2024-11-26T09:56:27,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987144 2024-11-26T09:56:27,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987138 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987144 2024-11-26T09:56:27,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,150 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,150 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=850, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,150 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987144) roll requested 2024-11-26T09:56:27,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987151 2024-11-26T09:56:27,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987144 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987151 2024-11-26T09:56:27,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,157 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,157 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=851, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,157 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987151) roll requested 2024-11-26T09:56:27,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987157 2024-11-26T09:56:27,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987151 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987157 2024-11-26T09:56:27,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,163 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,164 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=852, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,164 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987157) roll requested 2024-11-26T09:56:27,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987164 2024-11-26T09:56:27,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987157 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987164 2024-11-26T09:56:27,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,170 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,170 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=853, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,170 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987164) roll requested 2024-11-26T09:56:27,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987170 2024-11-26T09:56:27,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987164 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987170 2024-11-26T09:56:27,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,176 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,176 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=854, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,176 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987170) roll requested 2024-11-26T09:56:27,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987176 2024-11-26T09:56:27,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987170 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987176 2024-11-26T09:56:27,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,182 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,183 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=855, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,183 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987176) roll requested 2024-11-26T09:56:27,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987183 2024-11-26T09:56:27,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987176 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987183 2024-11-26T09:56:27,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,189 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,189 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=856, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987183) roll requested 2024-11-26T09:56:27,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987189 2024-11-26T09:56:27,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987183 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987189 2024-11-26T09:56:27,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=857, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987189) roll requested 2024-11-26T09:56:27,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987196 2024-11-26T09:56:27,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987196 2024-11-26T09:56:27,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,202 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,202 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=858, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,202 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987196) roll requested 2024-11-26T09:56:27,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987202 2024-11-26T09:56:27,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987202 2024-11-26T09:56:27,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,208 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,208 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=859, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,208 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987202) roll requested 2024-11-26T09:56:27,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987208 2024-11-26T09:56:27,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987202 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987208 2024-11-26T09:56:27,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,215 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,215 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=860, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,215 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987208) roll requested 2024-11-26T09:56:27,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987215 2024-11-26T09:56:27,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987208 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987215 2024-11-26T09:56:27,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,221 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,222 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=861, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,222 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987215) roll requested 2024-11-26T09:56:27,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987222 2024-11-26T09:56:27,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987215 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987222 2024-11-26T09:56:27,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,228 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,228 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=862, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987222) roll requested 2024-11-26T09:56:27,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987229 2024-11-26T09:56:27,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987222 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987229 2024-11-26T09:56:27,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,235 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,236 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=863, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,236 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987229) roll requested 2024-11-26T09:56:27,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987236 2024-11-26T09:56:27,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987236 2024-11-26T09:56:27,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,242 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,242 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=864, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,242 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987236) roll requested 2024-11-26T09:56:27,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987242 2024-11-26T09:56:27,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987236 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987242 2024-11-26T09:56:27,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,248 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,248 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=865, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,248 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987242) roll requested 2024-11-26T09:56:27,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987248 2024-11-26T09:56:27,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987242 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987248 2024-11-26T09:56:27,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,254 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,255 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=866, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,255 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987248) roll requested 2024-11-26T09:56:27,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987255 2024-11-26T09:56:27,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987248 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987255 2024-11-26T09:56:27,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,261 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,261 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=867, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987255) roll requested 2024-11-26T09:56:27,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987261 2024-11-26T09:56:27,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987255 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987261 2024-11-26T09:56:27,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,267 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=868, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987261) roll requested 2024-11-26T09:56:27,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987268 2024-11-26T09:56:27,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987268 2024-11-26T09:56:27,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,274 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,274 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=869, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,274 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987268) roll requested 2024-11-26T09:56:27,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987274 2024-11-26T09:56:27,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987268 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987274 2024-11-26T09:56:27,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,280 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,281 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=870, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,281 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987274) roll requested 2024-11-26T09:56:27,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987281 2024-11-26T09:56:27,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987274 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987281 2024-11-26T09:56:27,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,287 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,287 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=871, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,287 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987281) roll requested 2024-11-26T09:56:27,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987287 2024-11-26T09:56:27,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987281 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987287 2024-11-26T09:56:27,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,294 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,294 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=872, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,294 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987287) roll requested 2024-11-26T09:56:27,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987294 2024-11-26T09:56:27,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987287 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987294 2024-11-26T09:56:27,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,301 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,301 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=873, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,301 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987294) roll requested 2024-11-26T09:56:27,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987301 2024-11-26T09:56:27,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987294 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987301 2024-11-26T09:56:27,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,308 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=874, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,308 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987301) roll requested 2024-11-26T09:56:27,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987308 2024-11-26T09:56:27,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987308 2024-11-26T09:56:27,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,314 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,314 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=875, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,314 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987308) roll requested 2024-11-26T09:56:27,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987314 2024-11-26T09:56:27,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987308 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987314 2024-11-26T09:56:27,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=876, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987314) roll requested 2024-11-26T09:56:27,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987321 2024-11-26T09:56:27,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987314 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987321 2024-11-26T09:56:27,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,327 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,327 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=877, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,327 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987321) roll requested 2024-11-26T09:56:27,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987327 2024-11-26T09:56:27,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987327 2024-11-26T09:56:27,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,334 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=878, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,334 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987327) roll requested 2024-11-26T09:56:27,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987334 2024-11-26T09:56:27,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987327 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987334 2024-11-26T09:56:27,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,340 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=879, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,340 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987334) roll requested 2024-11-26T09:56:27,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987340 2024-11-26T09:56:27,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987334 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987340 2024-11-26T09:56:27,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,346 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,347 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=880, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,347 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987340) roll requested 2024-11-26T09:56:27,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987347 2024-11-26T09:56:27,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987340 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987347 2024-11-26T09:56:27,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,353 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,353 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=881, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,353 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987347) roll requested 2024-11-26T09:56:27,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987353 2024-11-26T09:56:27,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987347 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987353 2024-11-26T09:56:27,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,359 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,360 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=882, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,360 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987353) roll requested 2024-11-26T09:56:27,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987360 2024-11-26T09:56:27,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987353 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987360 2024-11-26T09:56:27,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,366 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,366 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=883, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,366 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987360) roll requested 2024-11-26T09:56:27,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987366 2024-11-26T09:56:27,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987360 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987366 2024-11-26T09:56:27,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,372 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,373 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=884, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,373 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987366) roll requested 2024-11-26T09:56:27,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987373 2024-11-26T09:56:27,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987366 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987373 2024-11-26T09:56:27,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,379 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,379 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=885, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,379 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987373) roll requested 2024-11-26T09:56:27,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987379 2024-11-26T09:56:27,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987379 2024-11-26T09:56:27,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,385 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,385 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=886, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987379) roll requested 2024-11-26T09:56:27,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987385 2024-11-26T09:56:27,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987379 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987385 2024-11-26T09:56:27,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,391 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,392 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=887, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,392 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987385) roll requested 2024-11-26T09:56:27,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987392 2024-11-26T09:56:27,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987392 2024-11-26T09:56:27,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,398 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,398 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=888, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,398 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987392) roll requested 2024-11-26T09:56:27,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987398 2024-11-26T09:56:27,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987392 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987398 2024-11-26T09:56:27,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,404 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,404 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=889, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,404 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987398) roll requested 2024-11-26T09:56:27,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987404 2024-11-26T09:56:27,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987398 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987404 2024-11-26T09:56:27,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,410 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,411 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=890, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,411 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987404) roll requested 2024-11-26T09:56:27,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987411 2024-11-26T09:56:27,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987404 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987411 2024-11-26T09:56:27,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,417 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,417 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=891, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987411) roll requested 2024-11-26T09:56:27,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987417 2024-11-26T09:56:27,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987411 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987417 2024-11-26T09:56:27,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=892, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987417) roll requested 2024-11-26T09:56:27,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987423 2024-11-26T09:56:27,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987423 2024-11-26T09:56:27,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,430 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,430 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=893, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,430 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987423) roll requested 2024-11-26T09:56:27,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987430 2024-11-26T09:56:27,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987430 2024-11-26T09:56:27,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,438 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,438 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=894, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,438 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987430) roll requested 2024-11-26T09:56:27,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987438 2024-11-26T09:56:27,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987430 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987438 2024-11-26T09:56:27,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,444 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,445 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=895, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,445 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987438) roll requested 2024-11-26T09:56:27,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987445 2024-11-26T09:56:27,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987438 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987445 2024-11-26T09:56:27,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,451 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,451 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=896, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,451 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987445) roll requested 2024-11-26T09:56:27,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987451 2024-11-26T09:56:27,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987445 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987451 2024-11-26T09:56:27,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,458 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,458 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=897, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,458 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987451) roll requested 2024-11-26T09:56:27,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987458 2024-11-26T09:56:27,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987451 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987458 2024-11-26T09:56:27,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,464 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=898, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987458) roll requested 2024-11-26T09:56:27,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987464 2024-11-26T09:56:27,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987458 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987464 2024-11-26T09:56:27,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,470 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,471 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=899, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,471 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987464) roll requested 2024-11-26T09:56:27,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987471 2024-11-26T09:56:27,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987471 2024-11-26T09:56:27,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,477 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=900, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987471) roll requested 2024-11-26T09:56:27,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987477 2024-11-26T09:56:27,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987471 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987477 2024-11-26T09:56:27,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,483 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,483 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=901, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,483 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987477) roll requested 2024-11-26T09:56:27,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987483 2024-11-26T09:56:27,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987483 2024-11-26T09:56:27,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,490 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,490 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=902, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,490 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987483) roll requested 2024-11-26T09:56:27,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987490 2024-11-26T09:56:27,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987483 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987490 2024-11-26T09:56:27,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,496 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,496 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=903, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,496 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987490) roll requested 2024-11-26T09:56:27,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987496 2024-11-26T09:56:27,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987490 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987496 2024-11-26T09:56:27,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=904, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,503 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987496) roll requested 2024-11-26T09:56:27,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987503 2024-11-26T09:56:27,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987496 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987503 2024-11-26T09:56:27,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,509 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,509 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=905, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,509 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987503) roll requested 2024-11-26T09:56:27,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987509 2024-11-26T09:56:27,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987503 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987509 2024-11-26T09:56:27,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,515 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,516 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=906, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,516 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987509) roll requested 2024-11-26T09:56:27,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987516 2024-11-26T09:56:27,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987516 2024-11-26T09:56:27,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,522 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,522 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=907, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,522 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987516) roll requested 2024-11-26T09:56:27,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987522 2024-11-26T09:56:27,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987516 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987522 2024-11-26T09:56:27,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,529 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=908, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987522) roll requested 2024-11-26T09:56:27,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987529 2024-11-26T09:56:27,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987522 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987529 2024-11-26T09:56:27,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,535 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,535 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=909, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,535 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987529) roll requested 2024-11-26T09:56:27,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987535 2024-11-26T09:56:27,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987535 2024-11-26T09:56:27,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,541 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,541 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=910, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,541 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987535) roll requested 2024-11-26T09:56:27,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987541 2024-11-26T09:56:27,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987535 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987541 2024-11-26T09:56:27,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,548 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,548 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=911, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,548 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987541) roll requested 2024-11-26T09:56:27,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987548 2024-11-26T09:56:27,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987541 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987548 2024-11-26T09:56:27,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,554 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,554 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=912, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,554 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987548) roll requested 2024-11-26T09:56:27,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987554 2024-11-26T09:56:27,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987548 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987554 2024-11-26T09:56:27,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,561 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,561 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=913, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,561 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987554) roll requested 2024-11-26T09:56:27,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987561 2024-11-26T09:56:27,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987554 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987561 2024-11-26T09:56:27,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,567 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,567 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=914, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,567 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987561) roll requested 2024-11-26T09:56:27,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987567 2024-11-26T09:56:27,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987561 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987567 2024-11-26T09:56:27,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,573 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,574 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=915, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,574 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987567) roll requested 2024-11-26T09:56:27,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987574 2024-11-26T09:56:27,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987567 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987574 2024-11-26T09:56:27,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,580 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,580 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=916, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,580 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987574) roll requested 2024-11-26T09:56:27,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987580 2024-11-26T09:56:27,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987574 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987580 2024-11-26T09:56:27,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,587 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,587 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=917, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,587 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987580) roll requested 2024-11-26T09:56:27,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987587 2024-11-26T09:56:27,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987580 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987587 2024-11-26T09:56:27,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,595 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,595 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=918, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,595 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987587) roll requested 2024-11-26T09:56:27,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987595 2024-11-26T09:56:27,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987587 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987595 2024-11-26T09:56:27,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,601 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,602 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=919, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,602 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987595) roll requested 2024-11-26T09:56:27,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987602 2024-11-26T09:56:27,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987595 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987602 2024-11-26T09:56:27,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=920, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987602) roll requested 2024-11-26T09:56:27,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987608 2024-11-26T09:56:27,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987602 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987608 2024-11-26T09:56:27,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,615 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,615 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=921, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,615 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987608) roll requested 2024-11-26T09:56:27,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987615 2024-11-26T09:56:27,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987615 2024-11-26T09:56:27,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,621 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,622 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=922, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,622 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987615) roll requested 2024-11-26T09:56:27,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987622 2024-11-26T09:56:27,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987615 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987622 2024-11-26T09:56:27,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,628 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=923, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987622) roll requested 2024-11-26T09:56:27,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987628 2024-11-26T09:56:27,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987622 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987628 2024-11-26T09:56:27,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,635 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,635 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=924, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,635 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987628) roll requested 2024-11-26T09:56:27,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987635 2024-11-26T09:56:27,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987635 2024-11-26T09:56:27,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,642 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,642 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=925, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,642 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987635) roll requested 2024-11-26T09:56:27,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987642 2024-11-26T09:56:27,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987635 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987642 2024-11-26T09:56:27,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,648 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,648 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=926, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,648 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987642) roll requested 2024-11-26T09:56:27,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987648 2024-11-26T09:56:27,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987642 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987648 2024-11-26T09:56:27,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,655 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,655 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=927, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,655 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987648) roll requested 2024-11-26T09:56:27,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987655 2024-11-26T09:56:27,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987648 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987655 2024-11-26T09:56:27,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,661 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,661 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=928, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,661 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987655) roll requested 2024-11-26T09:56:27,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987661 2024-11-26T09:56:27,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987655 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987661 2024-11-26T09:56:27,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,667 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,668 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=929, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,668 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987661) roll requested 2024-11-26T09:56:27,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987668 2024-11-26T09:56:27,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987661 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987668 2024-11-26T09:56:27,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=930, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987668) roll requested 2024-11-26T09:56:27,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987674 2024-11-26T09:56:27,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987668 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987674 2024-11-26T09:56:27,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=931, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987674) roll requested 2024-11-26T09:56:27,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987681 2024-11-26T09:56:27,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987681 2024-11-26T09:56:27,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,687 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=932, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,688 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987681) roll requested 2024-11-26T09:56:27,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987688 2024-11-26T09:56:27,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987688 2024-11-26T09:56:27,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,694 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,694 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=933, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,694 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987688) roll requested 2024-11-26T09:56:27,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987694 2024-11-26T09:56:27,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987688 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987694 2024-11-26T09:56:27,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,700 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=934, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987694) roll requested 2024-11-26T09:56:27,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987701 2024-11-26T09:56:27,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987701 2024-11-26T09:56:27,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,707 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,707 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=935, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,707 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987701) roll requested 2024-11-26T09:56:27,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987707 2024-11-26T09:56:27,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987707 2024-11-26T09:56:27,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,714 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,714 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=936, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,714 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987707) roll requested 2024-11-26T09:56:27,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987714 2024-11-26T09:56:27,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987707 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987714 2024-11-26T09:56:27,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,720 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,721 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=937, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,721 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987714) roll requested 2024-11-26T09:56:27,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987721 2024-11-26T09:56:27,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987714 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987721 2024-11-26T09:56:27,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,727 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,727 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=938, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,727 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987721) roll requested 2024-11-26T09:56:27,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987727 2024-11-26T09:56:27,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987721 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987727 2024-11-26T09:56:27,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=939, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987727) roll requested 2024-11-26T09:56:27,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987734 2024-11-26T09:56:27,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987727 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987734 2024-11-26T09:56:27,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=940, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987734) roll requested 2024-11-26T09:56:27,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987740 2024-11-26T09:56:27,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987734 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987740 2024-11-26T09:56:27,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,747 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=941, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,747 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987740) roll requested 2024-11-26T09:56:27,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987747 2024-11-26T09:56:27,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987740 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987747 2024-11-26T09:56:27,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,753 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=942, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,753 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987747) roll requested 2024-11-26T09:56:27,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987753 2024-11-26T09:56:27,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987747 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987753 2024-11-26T09:56:27,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=943, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987753) roll requested 2024-11-26T09:56:27,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987760 2024-11-26T09:56:27,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987760 2024-11-26T09:56:27,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=944, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987760) roll requested 2024-11-26T09:56:27,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987766 2024-11-26T09:56:27,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987760 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987766 2024-11-26T09:56:27,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,772 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,772 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=945, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,772 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987766) roll requested 2024-11-26T09:56:27,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987772 2024-11-26T09:56:27,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987766 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987772 2024-11-26T09:56:27,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,778 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,779 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=946, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,779 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987772) roll requested 2024-11-26T09:56:27,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987779 2024-11-26T09:56:27,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987772 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987779 2024-11-26T09:56:27,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,786 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=947, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,786 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987779) roll requested 2024-11-26T09:56:27,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987786 2024-11-26T09:56:27,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987786 2024-11-26T09:56:27,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,792 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=948, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987786) roll requested 2024-11-26T09:56:27,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987793 2024-11-26T09:56:27,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987786 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987793 2024-11-26T09:56:27,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,798 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=949, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987793) roll requested 2024-11-26T09:56:27,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987799 2024-11-26T09:56:27,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987793 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987799 2024-11-26T09:56:27,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,805 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=950, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,806 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987799) roll requested 2024-11-26T09:56:27,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987806 2024-11-26T09:56:27,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987806 2024-11-26T09:56:27,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,812 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=951, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,812 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987806) roll requested 2024-11-26T09:56:27,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987812 2024-11-26T09:56:27,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987806 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987812 2024-11-26T09:56:27,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,818 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,819 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=952, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,819 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987812) roll requested 2024-11-26T09:56:27,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987819 2024-11-26T09:56:27,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987812 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987819 2024-11-26T09:56:27,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,825 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,825 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=953, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,825 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987819) roll requested 2024-11-26T09:56:27,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987826 2024-11-26T09:56:27,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987819 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987826 2024-11-26T09:56:27,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,833 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,833 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=954, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,833 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987826) roll requested 2024-11-26T09:56:27,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987833 2024-11-26T09:56:27,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987826 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987833 2024-11-26T09:56:27,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,840 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=955, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,840 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987833) roll requested 2024-11-26T09:56:27,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987840 2024-11-26T09:56:27,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987833 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987840 2024-11-26T09:56:27,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,846 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=956, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,846 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987840) roll requested 2024-11-26T09:56:27,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987846 2024-11-26T09:56:27,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987840 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987846 2024-11-26T09:56:27,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,853 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,853 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=957, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,853 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987846) roll requested 2024-11-26T09:56:27,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987853 2024-11-26T09:56:27,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987846 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987853 2024-11-26T09:56:27,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,859 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,859 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=958, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987853) roll requested 2024-11-26T09:56:27,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987860 2024-11-26T09:56:27,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987853 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987860 2024-11-26T09:56:27,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,866 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,866 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=959, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,866 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987860) roll requested 2024-11-26T09:56:27,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987866 2024-11-26T09:56:27,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987860 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987866 2024-11-26T09:56:27,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,874 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,874 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=960, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,874 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987866) roll requested 2024-11-26T09:56:27,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987874 2024-11-26T09:56:27,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987866 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987874 2024-11-26T09:56:27,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,881 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=961, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,881 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987874) roll requested 2024-11-26T09:56:27,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987881 2024-11-26T09:56:27,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987874 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987881 2024-11-26T09:56:27,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=962, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,888 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987881) roll requested 2024-11-26T09:56:27,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987888 2024-11-26T09:56:27,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987881 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987888 2024-11-26T09:56:27,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,894 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,894 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=963, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,894 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987888) roll requested 2024-11-26T09:56:27,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987894 2024-11-26T09:56:27,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987888 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987894 2024-11-26T09:56:27,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=964, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987894) roll requested 2024-11-26T09:56:27,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987900 2024-11-26T09:56:27,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987900 2024-11-26T09:56:27,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,907 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,907 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=965, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,907 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987900) roll requested 2024-11-26T09:56:27,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987907 2024-11-26T09:56:27,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987907 2024-11-26T09:56:27,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,913 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,914 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=966, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,914 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987907) roll requested 2024-11-26T09:56:27,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987914 2024-11-26T09:56:27,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987907 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987914 2024-11-26T09:56:27,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=967, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,920 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987914) roll requested 2024-11-26T09:56:27,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987920 2024-11-26T09:56:27,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987914 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987920 2024-11-26T09:56:27,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,926 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,926 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=968, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,926 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987920) roll requested 2024-11-26T09:56:27,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987926 2024-11-26T09:56:27,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987926 2024-11-26T09:56:27,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,933 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,933 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=969, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,933 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987926) roll requested 2024-11-26T09:56:27,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987933 2024-11-26T09:56:27,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987926 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987933 2024-11-26T09:56:27,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,939 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,940 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=970, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,940 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987933) roll requested 2024-11-26T09:56:27,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987940 2024-11-26T09:56:27,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987933 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987940 2024-11-26T09:56:27,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,946 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,946 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=971, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,946 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987940) roll requested 2024-11-26T09:56:27,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987946 2024-11-26T09:56:27,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987940 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987946 2024-11-26T09:56:27,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,953 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,953 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=972, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,953 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987946) roll requested 2024-11-26T09:56:27,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987953 2024-11-26T09:56:27,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987953 2024-11-26T09:56:27,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=973, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,959 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987953) roll requested 2024-11-26T09:56:27,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987960 2024-11-26T09:56:27,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987960 2024-11-26T09:56:27,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,966 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,966 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=974, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,966 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987960) roll requested 2024-11-26T09:56:27,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987966 2024-11-26T09:56:27,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987966 2024-11-26T09:56:27,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=975, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987966) roll requested 2024-11-26T09:56:27,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987972 2024-11-26T09:56:27,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987966 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987972 2024-11-26T09:56:27,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=976, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987972) roll requested 2024-11-26T09:56:27,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987979 2024-11-26T09:56:27,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987979 2024-11-26T09:56:27,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,985 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=977, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,985 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987979) roll requested 2024-11-26T09:56:27,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987985 2024-11-26T09:56:27,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987985 2024-11-26T09:56:27,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,992 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,992 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=978, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,992 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987985) roll requested 2024-11-26T09:56:27,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987992 2024-11-26T09:56:27,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:27,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987992 2024-11-26T09:56:27,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:27,998 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:27,999 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=979, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:27,999 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:27,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987992) roll requested 2024-11-26T09:56:27,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614987999 2024-11-26T09:56:28,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987992 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987999 2024-11-26T09:56:28,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,007 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=980, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614987999) roll requested 2024-11-26T09:56:28,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988007 2024-11-26T09:56:28,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614987999 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988007 2024-11-26T09:56:28,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,014 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=981, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988007) roll requested 2024-11-26T09:56:28,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988014 2024-11-26T09:56:28,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988014 2024-11-26T09:56:28,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,020 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,020 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=982, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,020 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988014) roll requested 2024-11-26T09:56:28,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988020 2024-11-26T09:56:28,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988020 2024-11-26T09:56:28,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,027 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,027 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=983, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,027 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988020) roll requested 2024-11-26T09:56:28,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988027 2024-11-26T09:56:28,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988020 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988027 2024-11-26T09:56:28,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,033 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,034 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=984, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,034 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988027) roll requested 2024-11-26T09:56:28,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988034 2024-11-26T09:56:28,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988027 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988034 2024-11-26T09:56:28,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,040 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,040 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=985, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,040 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988034) roll requested 2024-11-26T09:56:28,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988040 2024-11-26T09:56:28,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988034 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988040 2024-11-26T09:56:28,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,046 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,046 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=986, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,046 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988040) roll requested 2024-11-26T09:56:28,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988046 2024-11-26T09:56:28,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988040 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988046 2024-11-26T09:56:28,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,053 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,053 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=987, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,053 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988046) roll requested 2024-11-26T09:56:28,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988053 2024-11-26T09:56:28,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988046 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988053 2024-11-26T09:56:28,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,059 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,059 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=988, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,059 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988053) roll requested 2024-11-26T09:56:28,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988059 2024-11-26T09:56:28,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988053 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988059 2024-11-26T09:56:28,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,065 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,066 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=989, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,066 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988059) roll requested 2024-11-26T09:56:28,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988066 2024-11-26T09:56:28,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988059 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988066 2024-11-26T09:56:28,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,072 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=990, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,072 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988066) roll requested 2024-11-26T09:56:28,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988072 2024-11-26T09:56:28,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988066 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988072 2024-11-26T09:56:28,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,079 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=991, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,079 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988072) roll requested 2024-11-26T09:56:28,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988079 2024-11-26T09:56:28,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988072 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988079 2024-11-26T09:56:28,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,085 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,085 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=992, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,085 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988079) roll requested 2024-11-26T09:56:28,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988085 2024-11-26T09:56:28,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988085 2024-11-26T09:56:28,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,091 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,092 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=993, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,092 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988085) roll requested 2024-11-26T09:56:28,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988092 2024-11-26T09:56:28,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988092 2024-11-26T09:56:28,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,098 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,098 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=994, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,099 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988092) roll requested 2024-11-26T09:56:28,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988099 2024-11-26T09:56:28,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988092 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988099 2024-11-26T09:56:28,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,105 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,105 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=995, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988099) roll requested 2024-11-26T09:56:28,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988105 2024-11-26T09:56:28,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988099 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988105 2024-11-26T09:56:28,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,112 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=996, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,112 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988105) roll requested 2024-11-26T09:56:28,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988112 2024-11-26T09:56:28,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988112 2024-11-26T09:56:28,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,118 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,118 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=997, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,118 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988112) roll requested 2024-11-26T09:56:28,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988118 2024-11-26T09:56:28,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988112 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988118 2024-11-26T09:56:28,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,124 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,125 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=998, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,125 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988118) roll requested 2024-11-26T09:56:28,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988125 2024-11-26T09:56:28,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988118 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988125 2024-11-26T09:56:28,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,131 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,131 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=999, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,131 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988125) roll requested 2024-11-26T09:56:28,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988131 2024-11-26T09:56:28,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988125 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988131 2024-11-26T09:56:28,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,138 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,138 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1000, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,138 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988131) roll requested 2024-11-26T09:56:28,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988139 2024-11-26T09:56:28,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988131 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988139 2024-11-26T09:56:28,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,150 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,150 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1001, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,150 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988139) roll requested 2024-11-26T09:56:28,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988151 2024-11-26T09:56:28,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988139 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988151 2024-11-26T09:56:28,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,160 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,160 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1002, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,160 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988151) roll requested 2024-11-26T09:56:28,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988160 2024-11-26T09:56:28,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988151 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988160 2024-11-26T09:56:28,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,168 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,168 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1003, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,168 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988160) roll requested 2024-11-26T09:56:28,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988168 2024-11-26T09:56:28,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988160 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988168 2024-11-26T09:56:28,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,176 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,177 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1004, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,177 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988168) roll requested 2024-11-26T09:56:28,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988177 2024-11-26T09:56:28,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988168 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988177 2024-11-26T09:56:28,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1005, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988177) roll requested 2024-11-26T09:56:28,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988185 2024-11-26T09:56:28,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988185 2024-11-26T09:56:28,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,191 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,191 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1006, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,191 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988185) roll requested 2024-11-26T09:56:28,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988191 2024-11-26T09:56:28,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988191 2024-11-26T09:56:28,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,198 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1007, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988191) roll requested 2024-11-26T09:56:28,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988198 2024-11-26T09:56:28,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988191 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988198 2024-11-26T09:56:28,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,205 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1008, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,205 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988198) roll requested 2024-11-26T09:56:28,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988205 2024-11-26T09:56:28,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988198 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988205 2024-11-26T09:56:28,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,211 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1009, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,212 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988205) roll requested 2024-11-26T09:56:28,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988212 2024-11-26T09:56:28,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988205 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988212 2024-11-26T09:56:28,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,218 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1010, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988212) roll requested 2024-11-26T09:56:28,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988218 2024-11-26T09:56:28,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988212 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988218 2024-11-26T09:56:28,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,225 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,226 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1011, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,226 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988218) roll requested 2024-11-26T09:56:28,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988226 2024-11-26T09:56:28,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988218 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988226 2024-11-26T09:56:28,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,232 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1012, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,232 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988226) roll requested 2024-11-26T09:56:28,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988232 2024-11-26T09:56:28,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988232 2024-11-26T09:56:28,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,239 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1013, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,239 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988232) roll requested 2024-11-26T09:56:28,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988239 2024-11-26T09:56:28,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988232 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988239 2024-11-26T09:56:28,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,245 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,245 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1014, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988239) roll requested 2024-11-26T09:56:28,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988245 2024-11-26T09:56:28,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988239 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988245 2024-11-26T09:56:28,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1015, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988245) roll requested 2024-11-26T09:56:28,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988252 2024-11-26T09:56:28,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988252 2024-11-26T09:56:28,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,258 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,259 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1016, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,259 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988252) roll requested 2024-11-26T09:56:28,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988259 2024-11-26T09:56:28,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988252 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988259 2024-11-26T09:56:28,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,265 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,265 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1017, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,266 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988259) roll requested 2024-11-26T09:56:28,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988266 2024-11-26T09:56:28,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988259 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988266 2024-11-26T09:56:28,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1018, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,272 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988266) roll requested 2024-11-26T09:56:28,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988272 2024-11-26T09:56:28,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988266 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988272 2024-11-26T09:56:28,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,278 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,279 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1019, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,279 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988272) roll requested 2024-11-26T09:56:28,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988279 2024-11-26T09:56:28,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988279 2024-11-26T09:56:28,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1020, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988279) roll requested 2024-11-26T09:56:28,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988285 2024-11-26T09:56:28,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988279 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988285 2024-11-26T09:56:28,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1021, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988285) roll requested 2024-11-26T09:56:28,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988292 2024-11-26T09:56:28,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988292 2024-11-26T09:56:28,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,298 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,299 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1022, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,299 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988292) roll requested 2024-11-26T09:56:28,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988299 2024-11-26T09:56:28,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988299 2024-11-26T09:56:28,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,305 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,306 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1023, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,306 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988299) roll requested 2024-11-26T09:56:28,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988306 2024-11-26T09:56:28,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988299 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988306 2024-11-26T09:56:28,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1024, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988306) roll requested 2024-11-26T09:56:28,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988312 2024-11-26T09:56:28,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988306 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988312 2024-11-26T09:56:28,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,318 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,318 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1025, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,319 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988312) roll requested 2024-11-26T09:56:28,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988319 2024-11-26T09:56:28,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988319 2024-11-26T09:56:28,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,325 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,325 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1026, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,325 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988319) roll requested 2024-11-26T09:56:28,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988325 2024-11-26T09:56:28,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988319 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988325 2024-11-26T09:56:28,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,331 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,332 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1027, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,332 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988325) roll requested 2024-11-26T09:56:28,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988332 2024-11-26T09:56:28,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988332 2024-11-26T09:56:28,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,338 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,338 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1028, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,338 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988332) roll requested 2024-11-26T09:56:28,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988338 2024-11-26T09:56:28,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988332 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988338 2024-11-26T09:56:28,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,345 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,345 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1029, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,345 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988338) roll requested 2024-11-26T09:56:28,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988345 2024-11-26T09:56:28,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988338 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988345 2024-11-26T09:56:28,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,352 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,352 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1030, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,352 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988345) roll requested 2024-11-26T09:56:28,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988352 2024-11-26T09:56:28,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988345 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988352 2024-11-26T09:56:28,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,358 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,358 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1031, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,358 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988352) roll requested 2024-11-26T09:56:28,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988358 2024-11-26T09:56:28,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988352 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988358 2024-11-26T09:56:28,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,365 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,365 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1032, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,365 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988358) roll requested 2024-11-26T09:56:28,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988365 2024-11-26T09:56:28,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988358 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988365 2024-11-26T09:56:28,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,371 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,371 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1033, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,371 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988365) roll requested 2024-11-26T09:56:28,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988371 2024-11-26T09:56:28,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988365 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988371 2024-11-26T09:56:28,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,378 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1034, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,378 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988371) roll requested 2024-11-26T09:56:28,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988378 2024-11-26T09:56:28,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988378 2024-11-26T09:56:28,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,384 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,385 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1035, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988378) roll requested 2024-11-26T09:56:28,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988385 2024-11-26T09:56:28,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988378 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988385 2024-11-26T09:56:28,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,391 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,391 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1036, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,391 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988385) roll requested 2024-11-26T09:56:28,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988391 2024-11-26T09:56:28,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988391 2024-11-26T09:56:28,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1037, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988391) roll requested 2024-11-26T09:56:28,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988397 2024-11-26T09:56:28,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988391 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988397 2024-11-26T09:56:28,404 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,404 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1038, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,404 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988397) roll requested 2024-11-26T09:56:28,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988404 2024-11-26T09:56:28,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988404 2024-11-26T09:56:28,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,410 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,411 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1039, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,411 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988404) roll requested 2024-11-26T09:56:28,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988411 2024-11-26T09:56:28,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988404 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988411 2024-11-26T09:56:28,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,417 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,418 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1040, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,418 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988411) roll requested 2024-11-26T09:56:28,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988418 2024-11-26T09:56:28,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988411 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988418 2024-11-26T09:56:28,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1041, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988418) roll requested 2024-11-26T09:56:28,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988424 2024-11-26T09:56:28,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988418 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988424 2024-11-26T09:56:28,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,431 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1042, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988424) roll requested 2024-11-26T09:56:28,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988431 2024-11-26T09:56:28,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988431 2024-11-26T09:56:28,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,437 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,437 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1043, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,438 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988431) roll requested 2024-11-26T09:56:28,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988438 2024-11-26T09:56:28,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988438 2024-11-26T09:56:28,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,444 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,444 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1044, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,444 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988438) roll requested 2024-11-26T09:56:28,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988445 2024-11-26T09:56:28,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988438 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988445 2024-11-26T09:56:28,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,451 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,451 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1045, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,451 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988445) roll requested 2024-11-26T09:56:28,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988452 2024-11-26T09:56:28,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988445 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988452 2024-11-26T09:56:28,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,458 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,458 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1046, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,458 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988452) roll requested 2024-11-26T09:56:28,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988458 2024-11-26T09:56:28,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988452 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988458 2024-11-26T09:56:28,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,464 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,465 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1047, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,465 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988458) roll requested 2024-11-26T09:56:28,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988465 2024-11-26T09:56:28,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988458 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988465 2024-11-26T09:56:28,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,471 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,471 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1048, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,471 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988465) roll requested 2024-11-26T09:56:28,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988471 2024-11-26T09:56:28,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988465 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988471 2024-11-26T09:56:28,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,478 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1049, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988471) roll requested 2024-11-26T09:56:28,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988478 2024-11-26T09:56:28,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988471 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988478 2024-11-26T09:56:28,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1050, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988478) roll requested 2024-11-26T09:56:28,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988485 2024-11-26T09:56:28,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988485 2024-11-26T09:56:28,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,491 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,491 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1051, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,491 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988485) roll requested 2024-11-26T09:56:28,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988491 2024-11-26T09:56:28,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988491 2024-11-26T09:56:28,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,497 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,497 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1052, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988491) roll requested 2024-11-26T09:56:28,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988497 2024-11-26T09:56:28,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988491 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988497 2024-11-26T09:56:28,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1053, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988497) roll requested 2024-11-26T09:56:28,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988504 2024-11-26T09:56:28,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988504 2024-11-26T09:56:28,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,510 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,510 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1054, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,510 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988504) roll requested 2024-11-26T09:56:28,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988510 2024-11-26T09:56:28,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988510 2024-11-26T09:56:28,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,516 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,517 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1055, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,517 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988510) roll requested 2024-11-26T09:56:28,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988517 2024-11-26T09:56:28,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988510 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988517 2024-11-26T09:56:28,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,523 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1056, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,523 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988517) roll requested 2024-11-26T09:56:28,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988523 2024-11-26T09:56:28,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988517 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988523 2024-11-26T09:56:28,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,529 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,529 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1057, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988523) roll requested 2024-11-26T09:56:28,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988529 2024-11-26T09:56:28,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988523 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988529 2024-11-26T09:56:28,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,536 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,536 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1058, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,536 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988529) roll requested 2024-11-26T09:56:28,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988536 2024-11-26T09:56:28,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988536 2024-11-26T09:56:28,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,544 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1059, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,544 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988536) roll requested 2024-11-26T09:56:28,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988544 2024-11-26T09:56:28,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988536 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988544 2024-11-26T09:56:28,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,550 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,551 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1060, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,551 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988544) roll requested 2024-11-26T09:56:28,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988551 2024-11-26T09:56:28,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988544 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988551 2024-11-26T09:56:28,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,557 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1061, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988551) roll requested 2024-11-26T09:56:28,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988557 2024-11-26T09:56:28,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988551 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988557 2024-11-26T09:56:28,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1062, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,564 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988557) roll requested 2024-11-26T09:56:28,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988564 2024-11-26T09:56:28,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988557 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988564 2024-11-26T09:56:28,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,570 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,570 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1063, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,570 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988564) roll requested 2024-11-26T09:56:28,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988571 2024-11-26T09:56:28,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988571 2024-11-26T09:56:28,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,580 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,580 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1064, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,580 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988571) roll requested 2024-11-26T09:56:28,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988581 2024-11-26T09:56:28,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988571 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988581 2024-11-26T09:56:28,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,587 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,587 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1065, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,587 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988581) roll requested 2024-11-26T09:56:28,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988587 2024-11-26T09:56:28,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988581 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988587 2024-11-26T09:56:28,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,595 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,596 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1066, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,596 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988587) roll requested 2024-11-26T09:56:28,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988596 2024-11-26T09:56:28,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988587 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988596 2024-11-26T09:56:28,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,602 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,602 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1067, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,602 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988596) roll requested 2024-11-26T09:56:28,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988602 2024-11-26T09:56:28,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988596 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988602 2024-11-26T09:56:28,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,609 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,609 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1068, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,609 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988602) roll requested 2024-11-26T09:56:28,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988609 2024-11-26T09:56:28,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988602 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988609 2024-11-26T09:56:28,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,615 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,615 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1069, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,615 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988609) roll requested 2024-11-26T09:56:28,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988615 2024-11-26T09:56:28,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988615 2024-11-26T09:56:28,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,622 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,622 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1070, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,622 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988615) roll requested 2024-11-26T09:56:28,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988622 2024-11-26T09:56:28,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988615 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988622 2024-11-26T09:56:28,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,629 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1071, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,629 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988622) roll requested 2024-11-26T09:56:28,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988629 2024-11-26T09:56:28,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988622 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988629 2024-11-26T09:56:28,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,635 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,635 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1072, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,635 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988629) roll requested 2024-11-26T09:56:28,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988635 2024-11-26T09:56:28,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988635 2024-11-26T09:56:28,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,642 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,642 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1073, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,642 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988635) roll requested 2024-11-26T09:56:28,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988642 2024-11-26T09:56:28,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988635 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988642 2024-11-26T09:56:28,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,648 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,648 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1074, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,649 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988642) roll requested 2024-11-26T09:56:28,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988649 2024-11-26T09:56:28,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988642 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988649 2024-11-26T09:56:28,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,655 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,655 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1075, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,655 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988649) roll requested 2024-11-26T09:56:28,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988655 2024-11-26T09:56:28,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988649 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988655 2024-11-26T09:56:28,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,661 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,661 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1076, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,662 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988655) roll requested 2024-11-26T09:56:28,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988662 2024-11-26T09:56:28,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988655 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988662 2024-11-26T09:56:28,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,668 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,668 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1077, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,668 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988662) roll requested 2024-11-26T09:56:28,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988668 2024-11-26T09:56:28,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988662 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988668 2024-11-26T09:56:28,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,675 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1078, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,675 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988668) roll requested 2024-11-26T09:56:28,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988675 2024-11-26T09:56:28,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988668 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988675 2024-11-26T09:56:28,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,681 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1079, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988675) roll requested 2024-11-26T09:56:28,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988681 2024-11-26T09:56:28,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988675 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988681 2024-11-26T09:56:28,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,688 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,688 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1080, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,688 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988681) roll requested 2024-11-26T09:56:28,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988688 2024-11-26T09:56:28,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988688 2024-11-26T09:56:28,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,694 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,694 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1081, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,694 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988688) roll requested 2024-11-26T09:56:28,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988694 2024-11-26T09:56:28,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988688 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988694 2024-11-26T09:56:28,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1082, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988694) roll requested 2024-11-26T09:56:28,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988701 2024-11-26T09:56:28,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988701 2024-11-26T09:56:28,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,707 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1083, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988701) roll requested 2024-11-26T09:56:28,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988708 2024-11-26T09:56:28,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988708 2024-11-26T09:56:28,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,714 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,714 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1084, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,715 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988708) roll requested 2024-11-26T09:56:28,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988715 2024-11-26T09:56:28,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988715 2024-11-26T09:56:28,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,721 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,721 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1085, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,722 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988715) roll requested 2024-11-26T09:56:28,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988722 2024-11-26T09:56:28,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988715 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988722 2024-11-26T09:56:28,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,728 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,728 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1086, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,728 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988722) roll requested 2024-11-26T09:56:28,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988728 2024-11-26T09:56:28,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988722 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988728 2024-11-26T09:56:28,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,735 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1087, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,735 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988728) roll requested 2024-11-26T09:56:28,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988735 2024-11-26T09:56:28,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988728 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988735 2024-11-26T09:56:28,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,741 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,741 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1088, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,741 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988735) roll requested 2024-11-26T09:56:28,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988741 2024-11-26T09:56:28,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988735 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988741 2024-11-26T09:56:28,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,747 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1089, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,748 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988741) roll requested 2024-11-26T09:56:28,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988748 2024-11-26T09:56:28,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988748 2024-11-26T09:56:28,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,754 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,754 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1090, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,755 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988748) roll requested 2024-11-26T09:56:28,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988755 2024-11-26T09:56:28,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988748 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988755 2024-11-26T09:56:28,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,761 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1091, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,761 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988755) roll requested 2024-11-26T09:56:28,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988761 2024-11-26T09:56:28,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988755 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988761 2024-11-26T09:56:28,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,768 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1092, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988761) roll requested 2024-11-26T09:56:28,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988768 2024-11-26T09:56:28,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988768 2024-11-26T09:56:28,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,774 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,774 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1093, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,774 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988768) roll requested 2024-11-26T09:56:28,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988775 2024-11-26T09:56:28,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988768 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988775 2024-11-26T09:56:28,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,781 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,781 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1094, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,781 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988775) roll requested 2024-11-26T09:56:28,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988781 2024-11-26T09:56:28,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988775 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988781 2024-11-26T09:56:28,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1095, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,789 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988781) roll requested 2024-11-26T09:56:28,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988789 2024-11-26T09:56:28,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988781 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988789 2024-11-26T09:56:28,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,795 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,795 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1096, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,795 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988789) roll requested 2024-11-26T09:56:28,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988795 2024-11-26T09:56:28,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988789 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988795 2024-11-26T09:56:28,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,802 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1097, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988795) roll requested 2024-11-26T09:56:28,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988802 2024-11-26T09:56:28,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988795 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988802 2024-11-26T09:56:28,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,809 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,809 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1098, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,809 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988802) roll requested 2024-11-26T09:56:28,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988809 2024-11-26T09:56:28,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988809 2024-11-26T09:56:28,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,815 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,815 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1099, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,816 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988809) roll requested 2024-11-26T09:56:28,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988816 2024-11-26T09:56:28,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988809 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988816 2024-11-26T09:56:28,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,822 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1100, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,822 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988816) roll requested 2024-11-26T09:56:28,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988822 2024-11-26T09:56:28,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988816 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988822 2024-11-26T09:56:28,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,829 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,829 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1101, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,829 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988822) roll requested 2024-11-26T09:56:28,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988829 2024-11-26T09:56:28,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988822 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988829 2024-11-26T09:56:28,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,836 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1102, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,836 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988829) roll requested 2024-11-26T09:56:28,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988836 2024-11-26T09:56:28,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988829 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988836 2024-11-26T09:56:28,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,842 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,843 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1103, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,843 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988836) roll requested 2024-11-26T09:56:28,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988843 2024-11-26T09:56:28,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988836 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988843 2024-11-26T09:56:28,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,849 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,850 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1104, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,850 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988843) roll requested 2024-11-26T09:56:28,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988850 2024-11-26T09:56:28,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988843 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988850 2024-11-26T09:56:28,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,857 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1105, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,857 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988850) roll requested 2024-11-26T09:56:28,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988857 2024-11-26T09:56:28,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988850 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988857 2024-11-26T09:56:28,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,863 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1106, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,864 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988857) roll requested 2024-11-26T09:56:28,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988864 2024-11-26T09:56:28,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988857 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988864 2024-11-26T09:56:28,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,871 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1107, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,871 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988864) roll requested 2024-11-26T09:56:28,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988871 2024-11-26T09:56:28,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988864 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988871 2024-11-26T09:56:28,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,879 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,879 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1108, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,879 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988871) roll requested 2024-11-26T09:56:28,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988879 2024-11-26T09:56:28,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988871 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988879 2024-11-26T09:56:28,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,886 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,886 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1109, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,886 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988879) roll requested 2024-11-26T09:56:28,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988886 2024-11-26T09:56:28,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988879 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988886 2024-11-26T09:56:28,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1110, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,894 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988886) roll requested 2024-11-26T09:56:28,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988894 2024-11-26T09:56:28,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988886 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988894 2024-11-26T09:56:28,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1111, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988894) roll requested 2024-11-26T09:56:28,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988901 2024-11-26T09:56:28,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988901 2024-11-26T09:56:28,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,907 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,907 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1112, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,907 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988901) roll requested 2024-11-26T09:56:28,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988907 2024-11-26T09:56:28,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988907 2024-11-26T09:56:28,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,913 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,914 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1113, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,914 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988907) roll requested 2024-11-26T09:56:28,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988914 2024-11-26T09:56:28,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988907 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988914 2024-11-26T09:56:28,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1114, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,920 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988914) roll requested 2024-11-26T09:56:28,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988921 2024-11-26T09:56:28,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988914 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988921 2024-11-26T09:56:28,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,927 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,927 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1115, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,927 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988921) roll requested 2024-11-26T09:56:28,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988927 2024-11-26T09:56:28,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988927 2024-11-26T09:56:28,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,933 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,934 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1116, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,934 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988927) roll requested 2024-11-26T09:56:28,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988934 2024-11-26T09:56:28,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988927 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988934 2024-11-26T09:56:28,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,940 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,940 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1117, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,940 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988934) roll requested 2024-11-26T09:56:28,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988940 2024-11-26T09:56:28,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988934 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988940 2024-11-26T09:56:28,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,946 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,947 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1118, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,947 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988940) roll requested 2024-11-26T09:56:28,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988947 2024-11-26T09:56:28,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988940 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988947 2024-11-26T09:56:28,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,953 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,954 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1119, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,954 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988947) roll requested 2024-11-26T09:56:28,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988954 2024-11-26T09:56:28,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988947 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988954 2024-11-26T09:56:28,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1120, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988954) roll requested 2024-11-26T09:56:28,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988960 2024-11-26T09:56:28,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988954 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988960 2024-11-26T09:56:28,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,966 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,966 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1121, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,966 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988960) roll requested 2024-11-26T09:56:28,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988966 2024-11-26T09:56:28,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988966 2024-11-26T09:56:28,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,973 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1122, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988966) roll requested 2024-11-26T09:56:28,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988973 2024-11-26T09:56:28,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988966 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988973 2024-11-26T09:56:28,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,979 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1123, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988973) roll requested 2024-11-26T09:56:28,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988979 2024-11-26T09:56:28,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988973 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988979 2024-11-26T09:56:28,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,986 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1124, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988979) roll requested 2024-11-26T09:56:28,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988986 2024-11-26T09:56:28,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988986 2024-11-26T09:56:28,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,993 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1125, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988986) roll requested 2024-11-26T09:56:28,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988993 2024-11-26T09:56:28,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:28,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988986 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988993 2024-11-26T09:56:28,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:28,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:28,999 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1126, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:28,999 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:28,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988993) roll requested 2024-11-26T09:56:28,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614988999 2024-11-26T09:56:29,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988999 2024-11-26T09:56:29,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,005 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,006 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1127, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,006 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614988999) roll requested 2024-11-26T09:56:29,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989006 2024-11-26T09:56:29,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614988999 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989006 2024-11-26T09:56:29,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,012 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,012 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1128, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,012 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989006) roll requested 2024-11-26T09:56:29,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989012 2024-11-26T09:56:29,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989006 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989012 2024-11-26T09:56:29,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,019 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,019 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1129, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,019 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989012) roll requested 2024-11-26T09:56:29,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989019 2024-11-26T09:56:29,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989012 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989019 2024-11-26T09:56:29,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,026 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1130, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,026 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989019) roll requested 2024-11-26T09:56:29,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989026 2024-11-26T09:56:29,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989026 2024-11-26T09:56:29,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,032 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,032 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1131, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,032 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989026) roll requested 2024-11-26T09:56:29,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989032 2024-11-26T09:56:29,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989026 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989032 2024-11-26T09:56:29,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,038 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,038 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1132, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,038 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989032) roll requested 2024-11-26T09:56:29,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989038 2024-11-26T09:56:29,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989032 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989038 2024-11-26T09:56:29,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,045 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,045 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1133, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,045 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989038) roll requested 2024-11-26T09:56:29,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989045 2024-11-26T09:56:29,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989038 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989045 2024-11-26T09:56:29,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,051 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,052 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1134, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,052 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989045) roll requested 2024-11-26T09:56:29,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989052 2024-11-26T09:56:29,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989052 2024-11-26T09:56:29,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,058 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,058 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1135, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,058 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989052) roll requested 2024-11-26T09:56:29,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989058 2024-11-26T09:56:29,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989052 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989058 2024-11-26T09:56:29,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,065 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,065 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1136, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,065 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989058) roll requested 2024-11-26T09:56:29,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989065 2024-11-26T09:56:29,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989058 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989065 2024-11-26T09:56:29,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,072 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1137, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,072 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989065) roll requested 2024-11-26T09:56:29,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989072 2024-11-26T09:56:29,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989065 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989072 2024-11-26T09:56:29,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,078 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1138, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,079 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989072) roll requested 2024-11-26T09:56:29,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989079 2024-11-26T09:56:29,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989072 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989079 2024-11-26T09:56:29,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,085 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,085 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1139, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,085 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989079) roll requested 2024-11-26T09:56:29,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989085 2024-11-26T09:56:29,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989085 2024-11-26T09:56:29,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,092 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1140, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,092 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989085) roll requested 2024-11-26T09:56:29,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989092 2024-11-26T09:56:29,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989092 2024-11-26T09:56:29,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,099 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,099 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1141, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,099 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989092) roll requested 2024-11-26T09:56:29,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989099 2024-11-26T09:56:29,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989092 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989099 2024-11-26T09:56:29,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,105 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,106 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1142, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,106 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989099) roll requested 2024-11-26T09:56:29,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989106 2024-11-26T09:56:29,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989099 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989106 2024-11-26T09:56:29,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,113 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1143, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989106) roll requested 2024-11-26T09:56:29,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989113 2024-11-26T09:56:29,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989106 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989113 2024-11-26T09:56:29,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,119 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,119 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1144, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,119 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989113) roll requested 2024-11-26T09:56:29,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989119 2024-11-26T09:56:29,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989119 2024-11-26T09:56:29,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1145, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989119) roll requested 2024-11-26T09:56:29,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989126 2024-11-26T09:56:29,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989119 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989126 2024-11-26T09:56:29,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,132 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,132 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1146, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,132 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989126) roll requested 2024-11-26T09:56:29,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989132 2024-11-26T09:56:29,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989126 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989132 2024-11-26T09:56:29,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,139 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,139 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1147, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,139 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989132) roll requested 2024-11-26T09:56:29,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989139 2024-11-26T09:56:29,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989132 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989139 2024-11-26T09:56:29,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,145 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,146 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1148, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,146 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989139) roll requested 2024-11-26T09:56:29,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989146 2024-11-26T09:56:29,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989139 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989146 2024-11-26T09:56:29,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,152 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,152 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1149, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,152 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989146) roll requested 2024-11-26T09:56:29,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989152 2024-11-26T09:56:29,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989146 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989152 2024-11-26T09:56:29,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,159 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1150, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,159 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989152) roll requested 2024-11-26T09:56:29,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989159 2024-11-26T09:56:29,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989152 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989159 2024-11-26T09:56:29,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,168 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,168 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1151, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,168 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989159) roll requested 2024-11-26T09:56:29,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989168 2024-11-26T09:56:29,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989159 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989168 2024-11-26T09:56:29,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,177 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,177 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1152, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,177 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989168) roll requested 2024-11-26T09:56:29,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989177 2024-11-26T09:56:29,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989168 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989177 2024-11-26T09:56:29,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,185 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,185 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1153, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,185 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989177) roll requested 2024-11-26T09:56:29,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989185 2024-11-26T09:56:29,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989185 2024-11-26T09:56:29,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,192 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,192 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1154, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,192 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989185) roll requested 2024-11-26T09:56:29,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989192 2024-11-26T09:56:29,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989192 2024-11-26T09:56:29,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,198 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,198 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1155, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989192) roll requested 2024-11-26T09:56:29,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989199 2024-11-26T09:56:29,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989192 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989199 2024-11-26T09:56:29,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,205 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,205 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1156, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,205 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989199) roll requested 2024-11-26T09:56:29,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989205 2024-11-26T09:56:29,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989199 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989205 2024-11-26T09:56:29,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,211 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1157, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,211 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989205) roll requested 2024-11-26T09:56:29,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989212 2024-11-26T09:56:29,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989205 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989212 2024-11-26T09:56:29,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,218 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1158, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989212) roll requested 2024-11-26T09:56:29,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989218 2024-11-26T09:56:29,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989212 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989218 2024-11-26T09:56:29,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,224 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1159, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989218) roll requested 2024-11-26T09:56:29,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989225 2024-11-26T09:56:29,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989218 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989225 2024-11-26T09:56:29,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,231 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,231 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1160, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,231 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989225) roll requested 2024-11-26T09:56:29,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989231 2024-11-26T09:56:29,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989231 2024-11-26T09:56:29,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,237 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,238 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1161, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,238 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989231) roll requested 2024-11-26T09:56:29,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989238 2024-11-26T09:56:29,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989231 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989238 2024-11-26T09:56:29,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1162, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989238) roll requested 2024-11-26T09:56:29,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989245 2024-11-26T09:56:29,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989238 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989245 2024-11-26T09:56:29,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,251 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,251 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1163, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,251 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989245) roll requested 2024-11-26T09:56:29,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989251 2024-11-26T09:56:29,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989251 2024-11-26T09:56:29,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,257 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,257 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1164, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989251) roll requested 2024-11-26T09:56:29,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989258 2024-11-26T09:56:29,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989251 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989258 2024-11-26T09:56:29,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,264 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,264 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1165, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,264 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989258) roll requested 2024-11-26T09:56:29,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989264 2024-11-26T09:56:29,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989258 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989264 2024-11-26T09:56:29,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,270 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,271 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1166, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,271 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989264) roll requested 2024-11-26T09:56:29,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989271 2024-11-26T09:56:29,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989271 2024-11-26T09:56:29,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1167, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989271) roll requested 2024-11-26T09:56:29,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989278 2024-11-26T09:56:29,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989271 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989278 2024-11-26T09:56:29,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,284 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1168, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,284 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989278) roll requested 2024-11-26T09:56:29,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989284 2024-11-26T09:56:29,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989278 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989284 2024-11-26T09:56:29,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,290 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,291 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1169, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,291 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989284) roll requested 2024-11-26T09:56:29,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989291 2024-11-26T09:56:29,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989284 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989291 2024-11-26T09:56:29,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,297 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,297 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1170, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,298 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989291) roll requested 2024-11-26T09:56:29,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989298 2024-11-26T09:56:29,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989291 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989298 2024-11-26T09:56:29,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,304 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1171, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,304 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989298) roll requested 2024-11-26T09:56:29,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989304 2024-11-26T09:56:29,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989298 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989304 2024-11-26T09:56:29,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,310 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,311 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1172, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,311 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989304) roll requested 2024-11-26T09:56:29,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989311 2024-11-26T09:56:29,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989304 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989311 2024-11-26T09:56:29,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,317 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,317 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1173, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,317 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989311) roll requested 2024-11-26T09:56:29,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989317 2024-11-26T09:56:29,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989311 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989317 2024-11-26T09:56:29,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,323 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,324 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1174, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,324 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989317) roll requested 2024-11-26T09:56:29,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989324 2024-11-26T09:56:29,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989324 2024-11-26T09:56:29,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,330 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,330 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1175, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,330 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989324) roll requested 2024-11-26T09:56:29,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989331 2024-11-26T09:56:29,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989324 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989331 2024-11-26T09:56:29,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,337 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,338 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1176, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,338 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989331) roll requested 2024-11-26T09:56:29,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989338 2024-11-26T09:56:29,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989331 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989338 2024-11-26T09:56:29,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,344 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1177, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,344 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989338) roll requested 2024-11-26T09:56:29,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989344 2024-11-26T09:56:29,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989338 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989344 2024-11-26T09:56:29,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,350 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,351 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1178, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,351 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989344) roll requested 2024-11-26T09:56:29,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989351 2024-11-26T09:56:29,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989344 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989351 2024-11-26T09:56:29,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,357 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,358 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1179, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,358 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989351) roll requested 2024-11-26T09:56:29,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989358 2024-11-26T09:56:29,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989351 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989358 2024-11-26T09:56:29,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1180, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989358) roll requested 2024-11-26T09:56:29,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989364 2024-11-26T09:56:29,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989358 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989364 2024-11-26T09:56:29,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,370 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,371 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1181, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,371 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989364) roll requested 2024-11-26T09:56:29,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989371 2024-11-26T09:56:29,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989371 2024-11-26T09:56:29,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,377 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1182, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,377 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989371) roll requested 2024-11-26T09:56:29,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989377 2024-11-26T09:56:29,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989377 2024-11-26T09:56:29,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,383 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,384 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1183, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,384 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989377) roll requested 2024-11-26T09:56:29,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989384 2024-11-26T09:56:29,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989377 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989384 2024-11-26T09:56:29,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,390 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,391 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1184, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,391 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989384) roll requested 2024-11-26T09:56:29,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989391 2024-11-26T09:56:29,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989384 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989391 2024-11-26T09:56:29,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1185, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989391) roll requested 2024-11-26T09:56:29,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989397 2024-11-26T09:56:29,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989391 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989397 2024-11-26T09:56:29,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,404 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,404 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1186, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,404 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989397) roll requested 2024-11-26T09:56:29,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989404 2024-11-26T09:56:29,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989404 2024-11-26T09:56:29,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,410 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,411 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1187, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,411 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989404) roll requested 2024-11-26T09:56:29,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989411 2024-11-26T09:56:29,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989404 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989411 2024-11-26T09:56:29,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,417 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,417 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1188, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989411) roll requested 2024-11-26T09:56:29,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989417 2024-11-26T09:56:29,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989411 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989417 2024-11-26T09:56:29,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1189, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989417) roll requested 2024-11-26T09:56:29,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989424 2024-11-26T09:56:29,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989424 2024-11-26T09:56:29,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,430 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1190, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989424) roll requested 2024-11-26T09:56:29,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989431 2024-11-26T09:56:29,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989431 2024-11-26T09:56:29,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,437 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,437 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1191, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,437 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989431) roll requested 2024-11-26T09:56:29,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989437 2024-11-26T09:56:29,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989437 2024-11-26T09:56:29,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,444 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1192, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,444 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989437) roll requested 2024-11-26T09:56:29,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989444 2024-11-26T09:56:29,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989437 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989444 2024-11-26T09:56:29,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1193, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989444) roll requested 2024-11-26T09:56:29,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989450 2024-11-26T09:56:29,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989444 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989450 2024-11-26T09:56:29,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1194, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989450) roll requested 2024-11-26T09:56:29,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989457 2024-11-26T09:56:29,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989457 2024-11-26T09:56:29,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1195, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,464 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989457) roll requested 2024-11-26T09:56:29,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989464 2024-11-26T09:56:29,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989464 2024-11-26T09:56:29,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,470 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,471 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1196, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,471 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989464) roll requested 2024-11-26T09:56:29,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989471 2024-11-26T09:56:29,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989471 2024-11-26T09:56:29,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,477 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1197, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989471) roll requested 2024-11-26T09:56:29,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989477 2024-11-26T09:56:29,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989471 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989477 2024-11-26T09:56:29,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,483 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,483 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1198, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,483 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989477) roll requested 2024-11-26T09:56:29,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989483 2024-11-26T09:56:29,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989483 2024-11-26T09:56:29,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,490 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,490 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1199, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,490 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989483) roll requested 2024-11-26T09:56:29,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989490 2024-11-26T09:56:29,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989483 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989490 2024-11-26T09:56:29,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,497 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,497 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1200, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989490) roll requested 2024-11-26T09:56:29,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989497 2024-11-26T09:56:29,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989490 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989497 2024-11-26T09:56:29,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1201, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989497) roll requested 2024-11-26T09:56:29,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989504 2024-11-26T09:56:29,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989504 2024-11-26T09:56:29,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,510 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,510 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1202, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,510 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989504) roll requested 2024-11-26T09:56:29,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989510 2024-11-26T09:56:29,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989510 2024-11-26T09:56:29,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,517 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,517 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1203, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,517 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989510) roll requested 2024-11-26T09:56:29,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989517 2024-11-26T09:56:29,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989510 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989517 2024-11-26T09:56:29,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,523 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,524 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1204, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,524 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989517) roll requested 2024-11-26T09:56:29,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989524 2024-11-26T09:56:29,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989517 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989524 2024-11-26T09:56:29,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-26T09:56:29,530 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-26T09:56:29,530 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1205, max=55; forcing (partial) flush of 1 region(s): de4af5b36d9054a312d130701461afdb[MyCF] 2024-11-26T09:56:29,530 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of de4af5b36d9054a312d130701461afdb, because it is not online on us 2024-11-26T09:56:29,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732614989524) roll requested 2024-11-26T09:56:29,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732614989530 2024-11-26T09:56:29,536 ERROR [Time-limited test {}] wal.AbstractFSWAL(939): Failed log archiving for the log /a/b, java.io.IOException: throw archival exception at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archiveLogFile(TestFailedAppendAndSync.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.archive(AbstractFSWAL.java:934) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archive(TestFailedAppendAndSync.java:143) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync.testLockupAroundBadAssignSync(TestFailedAppendAndSync.java:280) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T09:56:29,537 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing de4af5b36d9054a312d130701461afdb, disabling compactions & flushes 2024-11-26T09:56:29,538 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testLockupAroundBadAssignSync,,1732614959147.de4af5b36d9054a312d130701461afdb. 2024-11-26T09:56:29,538 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testLockupAroundBadAssignSync,,1732614959147.de4af5b36d9054a312d130701461afdb. 2024-11-26T09:56:29,538 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testLockupAroundBadAssignSync,,1732614959147.de4af5b36d9054a312d130701461afdb. after waiting 0 ms 2024-11-26T09:56:29,538 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testLockupAroundBadAssignSync,,1732614959147.de4af5b36d9054a312d130701461afdb. 2024-11-26T09:56:29,540 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 83 in region testLockupAroundBadAssignSync,,1732614959147.de4af5b36d9054a312d130701461afdb. 2024-11-26T09:56:29,540 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testLockupAroundBadAssignSync,,1732614959147.de4af5b36d9054a312d130701461afdb. 2024-11-26T09:56:29,540 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for de4af5b36d9054a312d130701461afdb: Waiting for close lock at 1732614989537Disabling compacts and flushes for region at 1732614989537Disabling writes for close at 1732614989538 (+1 ms)Writing region close event to WAL at 1732614989539 (+1 ms)Closed at 1732614989540 (+1 ms) 2024-11-26T09:56:29,542 WARN [LogRoller {}] wal.AbstractProtobufLogWriter(199): Init output failed, path=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732614989530 java.io.InterruptedIOException: java.lang.InterruptedException at org.apache.hadoop.util.Shell.runCommand(Shell.java:1071) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1377) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1359) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:1116) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:454) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:414) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.createOutputStreamWithMode(RawLocalFileSystem.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:566) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:597) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSOutputSummer.(ChecksumFileSystem.java:637) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:721) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.createNonRecursive(ChecksumFileSystem.java:765) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem$FileSystemDataOutputStreamBuilder.build(FileSystem.java:4705) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.initOutput(ProtobufLogWriter.java:117) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:171) ~[classes/:?] at org.apache.hadoop.hbase.wal.FSHLogProvider.createWriter(FSHLogProvider.java:82) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.createWriterInstance(FSHLog.java:259) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:148) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriterInternal(AbstractFSWAL.java:1099) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$rollWriter$9(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.rollWriter(TestFailedAppendAndSync.java:129) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller$RollController.rollWal(AbstractWALRoller.java:311) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:212) ~[classes/:?] Caused by: java.lang.InterruptedException at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at java.lang.ProcessImpl.waitFor(ProcessImpl.java:434) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1061) ~[hadoop-common-3.4.1.jar:?] ... 27 more 2024-11-26T09:56:29,543 DEBUG [LogRoller {}] wal.FSHLogProvider(93): Error instantiating log writer. java.io.InterruptedIOException: java.lang.InterruptedException at org.apache.hadoop.util.Shell.runCommand(Shell.java:1071) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1377) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1359) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:1116) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:454) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:414) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.createOutputStreamWithMode(RawLocalFileSystem.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:566) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:597) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSOutputSummer.(ChecksumFileSystem.java:637) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:721) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.createNonRecursive(ChecksumFileSystem.java:765) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem$FileSystemDataOutputStreamBuilder.build(FileSystem.java:4705) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.initOutput(ProtobufLogWriter.java:117) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:171) ~[classes/:?] at org.apache.hadoop.hbase.wal.FSHLogProvider.createWriter(FSHLogProvider.java:82) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.createWriterInstance(FSHLog.java:259) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:148) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriterInternal(AbstractFSWAL.java:1099) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$rollWriter$9(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.rollWriter(TestFailedAppendAndSync.java:129) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller$RollController.rollWal(AbstractWALRoller.java:311) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:212) ~[classes/:?] Caused by: java.lang.InterruptedException at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at java.lang.ProcessImpl.waitFor(ProcessImpl.java:434) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1061) ~[hadoop-common-3.4.1.jar:?] ... 27 more 2024-11-26T09:56:29,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,544 ERROR [LogRoller {}] wal.AbstractWALRoller(227): Roll wal failed and waiting timeout, will not retry java.io.IOException: cannot get log writer at org.apache.hadoop.hbase.wal.FSHLogProvider.createWriter(FSHLogProvider.java:95) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.createWriterInstance(FSHLog.java:259) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:148) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriterInternal(AbstractFSWAL.java:1099) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$rollWriter$9(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.rollWriter(TestFailedAppendAndSync.java:129) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller$RollController.rollWal(AbstractWALRoller.java:311) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:212) ~[classes/:?] Caused by: java.io.InterruptedIOException: java.lang.InterruptedException at org.apache.hadoop.util.Shell.runCommand(Shell.java:1071) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1377) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1359) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:1116) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:454) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:414) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.createOutputStreamWithMode(RawLocalFileSystem.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:566) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:597) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSOutputSummer.(ChecksumFileSystem.java:637) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:721) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.createNonRecursive(ChecksumFileSystem.java:765) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem$FileSystemDataOutputStreamBuilder.build(FileSystem.java:4705) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.initOutput(ProtobufLogWriter.java:117) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:171) ~[classes/:?] at org.apache.hadoop.hbase.wal.FSHLogProvider.createWriter(FSHLogProvider.java:82) ~[classes/:?] ... 10 more Caused by: java.lang.InterruptedException at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at java.lang.ProcessImpl.waitFor(ProcessImpl.java:434) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1061) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1377) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1359) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:1116) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:454) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:414) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.createOutputStreamWithMode(RawLocalFileSystem.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:566) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:597) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSOutputSummer.(ChecksumFileSystem.java:637) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:721) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.createNonRecursive(ChecksumFileSystem.java:765) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem$FileSystemDataOutputStreamBuilder.build(FileSystem.java:4705) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.initOutput(ProtobufLogWriter.java:117) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:171) ~[classes/:?] at org.apache.hadoop.hbase.wal.FSHLogProvider.createWriter(FSHLogProvider.java:82) ~[classes/:?] ... 10 more 2024-11-26T09:56:29,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,544 INFO [LogRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T09:56:29,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T09:56:29,898 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: DodgyFSLog wal:(num 1732614989530) 2024-11-26T09:56:29,899 INFO [Time-limited test {}] regionserver.TestFailedAppendAndSync(105): Cleaning test directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6beeb79-0dcc-f14d-d22b-afdbf09746de 2024-11-26T09:56:30,475 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=22 (was 11) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=288 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=296 (was 162) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4269 (was 4076) - AvailableMemoryMB LEAK? -